2024-11-21 01:57:43,462 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-21 01:57:43,479 main DEBUG Took 0.013815 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 01:57:43,479 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 01:57:43,489 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 01:57:43,491 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 01:57:43,493 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:57:43,502 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 01:57:43,519 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:57:43,521 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:57:43,522 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:57:43,522 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:57:43,523 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:57:43,523 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:57:43,524 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:57:43,525 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:57:43,525 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:57:43,526 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:57:43,527 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:57:43,527 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:57:43,528 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:57:43,529 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:57:43,529 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:57:43,530 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:57:43,530 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:57:43,531 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:57:43,531 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:57:43,532 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:57:43,532 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:57:43,533 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:57:43,533 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:57:43,534 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 01:57:43,534 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:57:43,535 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 01:57:43,537 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 01:57:43,538 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 01:57:43,540 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 01:57:43,541 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 01:57:43,542 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 01:57:43,542 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 01:57:43,550 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 01:57:43,554 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 01:57:43,556 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 01:57:43,556 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 01:57:43,557 main DEBUG createAppenders(={Console}) 2024-11-21 01:57:43,558 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-21 01:57:43,558 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-21 01:57:43,559 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-21 01:57:43,559 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 01:57:43,560 main DEBUG OutputStream closed 2024-11-21 01:57:43,560 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 01:57:43,560 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 01:57:43,560 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-21 01:57:43,644 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 01:57:43,646 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 01:57:43,648 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 01:57:43,649 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 01:57:43,649 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 01:57:43,650 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 01:57:43,650 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 01:57:43,650 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 01:57:43,651 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 01:57:43,651 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 01:57:43,652 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 01:57:43,652 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 01:57:43,652 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 01:57:43,652 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 01:57:43,653 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 01:57:43,653 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 01:57:43,653 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 01:57:43,654 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 01:57:43,656 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 01:57:43,657 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-21 01:57:43,657 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 01:57:43,658 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-21T01:57:43,671 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-21 01:57:43,674 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 01:57:43,674 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T01:57:43,993 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c 2024-11-21T01:57:44,024 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=831, ProcessCount=11, AvailableMemoryMB=3329 2024-11-21T01:57:44,123 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T01:57:44,213 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:44,231 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/cluster_2e34fb2a-9b89-f090-a9a2-cf6fc7a8f7d2, deleteOnExit=true 2024-11-21T01:57:44,340 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/cluster_2e34fb2a-9b89-f090-a9a2-cf6fc7a8f7d2/zookeeper_0, clientPort=65204, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/cluster_2e34fb2a-9b89-f090-a9a2-cf6fc7a8f7d2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/cluster_2e34fb2a-9b89-f090-a9a2-cf6fc7a8f7d2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T01:57:44,354 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=65204 2024-11-21T01:57:44,355 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:44,446 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060 with version=8 2024-11-21T01:57:44,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/hbase-staging 2024-11-21T01:57:44,548 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T01:57:44,850 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b93ada835ef8:0 server-side Connection retries=45 2024-11-21T01:57:44,861 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:44,862 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:44,872 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T01:57:44,872 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:44,873 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T01:57:45,015 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T01:57:45,125 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T01:57:45,174 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T01:57:45,188 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T01:57:45,192 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T01:57:45,239 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21827 (auto-detected) 2024-11-21T01:57:45,240 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-21T01:57:45,264 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35875 2024-11-21T01:57:45,297 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35875 connecting to ZooKeeper ensemble=127.0.0.1:65204 2024-11-21T01:57:45,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:358750x0, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T01:57:45,336 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35875-0x10113037fa70000 connected 2024-11-21T01:57:45,365 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:45,366 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:45,376 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T01:57:45,381 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060, hbase.cluster.distributed=false 2024-11-21T01:57:45,413 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T01:57:45,421 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35875 2024-11-21T01:57:45,421 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35875 2024-11-21T01:57:45,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35875 2024-11-21T01:57:45,426 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35875 2024-11-21T01:57:45,427 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35875 2024-11-21T01:57:45,519 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b93ada835ef8:0 server-side Connection retries=45 2024-11-21T01:57:45,522 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:45,522 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:45,523 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T01:57:45,523 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:45,523 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T01:57:45,527 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T01:57:45,530 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T01:57:45,531 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36587 2024-11-21T01:57:45,533 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36587 connecting to ZooKeeper ensemble=127.0.0.1:65204 2024-11-21T01:57:45,534 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:45,535 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:45,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:365870x0, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T01:57:45,546 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:365870x0, quorum=127.0.0.1:65204, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T01:57:45,549 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36587-0x10113037fa70001 connected 2024-11-21T01:57:45,553 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T01:57:45,573 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/bucket.cache 2024-11-21T01:57:45,582 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T01:57:45,583 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T01:57:45,585 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T01:57:45,588 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T01:57:45,588 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T01:57:45,599 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T01:57:45,600 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T01:57:45,601 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/bucket.persistence is complete. 2024-11-21T01:57:45,609 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T01:57:45,612 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T01:57:45,624 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T01:57:45,629 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36587 2024-11-21T01:57:45,632 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36587 2024-11-21T01:57:45,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36587 2024-11-21T01:57:45,643 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36587 2024-11-21T01:57:45,645 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36587 2024-11-21T01:57:45,667 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b93ada835ef8:0 server-side Connection retries=45 2024-11-21T01:57:45,667 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:45,668 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:45,668 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T01:57:45,668 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:45,668 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T01:57:45,669 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T01:57:45,669 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T01:57:45,674 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37077 2024-11-21T01:57:45,676 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37077 connecting to ZooKeeper ensemble=127.0.0.1:65204 2024-11-21T01:57:45,677 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:45,678 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:45,684 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:370770x0, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T01:57:45,686 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:370770x0, quorum=127.0.0.1:65204, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T01:57:45,687 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T01:57:45,690 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37077-0x10113037fa70002 connected 2024-11-21T01:57:45,693 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/bucket.cache 2024-11-21T01:57:45,693 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T01:57:45,694 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T01:57:45,694 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T01:57:45,697 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T01:57:45,698 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T01:57:45,698 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T01:57:45,699 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/bucket.persistence is complete. 2024-11-21T01:57:45,699 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T01:57:45,701 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T01:57:45,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T01:57:45,706 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T01:57:45,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37077 2024-11-21T01:57:45,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37077 2024-11-21T01:57:45,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37077 2024-11-21T01:57:45,714 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37077 2024-11-21T01:57:45,715 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37077 2024-11-21T01:57:45,739 DEBUG [M:0;b93ada835ef8:35875 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b93ada835ef8:35875 2024-11-21T01:57:45,741 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b93ada835ef8,35875,1732154264631 2024-11-21T01:57:45,750 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T01:57:45,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T01:57:45,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T01:57:45,752 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b93ada835ef8,35875,1732154264631 2024-11-21T01:57:45,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T01:57:45,787 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T01:57:45,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:45,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:45,787 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:45,789 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T01:57:45,791 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b93ada835ef8,35875,1732154264631 from backup master directory 2024-11-21T01:57:45,794 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T01:57:45,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b93ada835ef8,35875,1732154264631 2024-11-21T01:57:45,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T01:57:45,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T01:57:45,795 WARN [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T01:57:45,796 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b93ada835ef8,35875,1732154264631 2024-11-21T01:57:45,799 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T01:57:45,801 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T01:57:45,819 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/hbase.id] with ID: d5d4c01c-9937-4836-9676-fc2e82ab4603 2024-11-21T01:57:45,819 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/.tmp/hbase.id 2024-11-21T01:57:45,827 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/hbase.id] 2024-11-21T01:57:45,879 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:45,886 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T01:57:45,912 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-21T01:57:45,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:45,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:45,917 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:45,944 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T01:57:45,947 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T01:57:46,004 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:57:46,011 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T01:57:46,082 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store 2024-11-21T01:57:46,106 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T01:57:46,110 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:57:46,112 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T01:57:46,112 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:57:46,112 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:57:46,115 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T01:57:46,115 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:57:46,115 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:57:46,117 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732154266111Disabling compacts and flushes for region at 1732154266111Disabling writes for close at 1732154266115 (+4 ms)Writing region close event to WAL at 1732154266115Closed at 1732154266115 2024-11-21T01:57:46,117 WARN [master/b93ada835ef8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/.initializing 2024-11-21T01:57:46,118 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/WALs/b93ada835ef8,35875,1732154264631 2024-11-21T01:57:46,132 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T01:57:46,145 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b93ada835ef8%2C35875%2C1732154264631, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/WALs/b93ada835ef8,35875,1732154264631, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/oldWALs, maxLogs=10 2024-11-21T01:57:46,182 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/WALs/b93ada835ef8,35875,1732154264631/b93ada835ef8%2C35875%2C1732154264631.1732154266149 2024-11-21T01:57:46,183 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T01:57:46,183 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:57:46,184 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:57:46,187 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:46,188 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:46,231 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:46,271 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T01:57:46,276 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:46,279 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:46,279 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:46,288 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T01:57:46,288 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:46,289 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:57:46,289 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:46,296 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T01:57:46,296 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:46,297 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:57:46,297 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:46,303 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T01:57:46,303 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:46,303 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:57:46,304 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:46,308 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:46,308 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:46,312 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:46,313 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:46,315 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T01:57:46,318 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:46,326 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T01:57:46,326 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59804843, jitterRate=-0.10883839428424835}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T01:57:46,334 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732154266203Initializing all the Stores at 1732154266204 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154266205 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732154266206 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732154266207 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732154266207Cleaning up temporary data from old regions at 1732154266313 (+106 ms)Region opened successfully at 1732154266333 (+20 ms) 2024-11-21T01:57:46,335 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T01:57:46,373 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24356e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b93ada835ef8/172.17.0.3:0 2024-11-21T01:57:46,408 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T01:57:46,420 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T01:57:46,421 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T01:57:46,425 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T01:57:46,426 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T01:57:46,433 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-21T01:57:46,434 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T01:57:46,463 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T01:57:46,472 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T01:57:46,476 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T01:57:46,479 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T01:57:46,480 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T01:57:46,482 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T01:57:46,484 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T01:57:46,488 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T01:57:46,490 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T01:57:46,491 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T01:57:46,493 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T01:57:46,513 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T01:57:46,515 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T01:57:46,528 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T01:57:46,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T01:57:46,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T01:57:46,529 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:46,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:46,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:46,534 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b93ada835ef8,35875,1732154264631, sessionid=0x10113037fa70000, setting cluster-up flag (Was=false) 2024-11-21T01:57:46,550 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:46,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:46,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:46,557 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T01:57:46,559 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b93ada835ef8,35875,1732154264631 2024-11-21T01:57:46,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:46,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:46,563 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:46,570 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T01:57:46,572 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b93ada835ef8,35875,1732154264631 2024-11-21T01:57:46,577 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T01:57:46,620 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(746): ClusterId : d5d4c01c-9937-4836-9676-fc2e82ab4603 2024-11-21T01:57:46,620 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(746): ClusterId : d5d4c01c-9937-4836-9676-fc2e82ab4603 2024-11-21T01:57:46,623 DEBUG [RS:0;b93ada835ef8:36587 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T01:57:46,623 DEBUG [RS:1;b93ada835ef8:37077 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T01:57:46,628 DEBUG [RS:0;b93ada835ef8:36587 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T01:57:46,628 DEBUG [RS:0;b93ada835ef8:36587 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T01:57:46,629 DEBUG [RS:1;b93ada835ef8:37077 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T01:57:46,629 DEBUG [RS:1;b93ada835ef8:37077 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T01:57:46,633 DEBUG [RS:0;b93ada835ef8:36587 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T01:57:46,633 DEBUG [RS:1;b93ada835ef8:37077 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T01:57:46,633 DEBUG [RS:1;b93ada835ef8:37077 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cabd15f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b93ada835ef8/172.17.0.3:0 2024-11-21T01:57:46,633 DEBUG [RS:0;b93ada835ef8:36587 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@783d42af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b93ada835ef8/172.17.0.3:0 2024-11-21T01:57:46,650 DEBUG [RS:1;b93ada835ef8:37077 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;b93ada835ef8:37077 2024-11-21T01:57:46,653 DEBUG [RS:0;b93ada835ef8:36587 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b93ada835ef8:36587 2024-11-21T01:57:46,655 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T01:57:46,655 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T01:57:46,655 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T01:57:46,655 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T01:57:46,655 DEBUG [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T01:57:46,655 DEBUG [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T01:57:46,658 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(2659): reportForDuty to master=b93ada835ef8,35875,1732154264631 with port=36587, startcode=1732154265478 2024-11-21T01:57:46,658 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(2659): reportForDuty to master=b93ada835ef8,35875,1732154264631 with port=37077, startcode=1732154265666 2024-11-21T01:57:46,671 DEBUG [RS:0;b93ada835ef8:36587 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T01:57:46,671 DEBUG [RS:1;b93ada835ef8:37077 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T01:57:46,675 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T01:57:46,687 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T01:57:46,697 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T01:57:46,703 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b93ada835ef8,35875,1732154264631 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T01:57:46,712 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b93ada835ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-21T01:57:46,712 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b93ada835ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-21T01:57:46,713 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b93ada835ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-21T01:57:46,714 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b93ada835ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-21T01:57:46,715 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b93ada835ef8:0, corePoolSize=10, maxPoolSize=10 2024-11-21T01:57:46,720 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,726 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b93ada835ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-21T01:57:46,727 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,738 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T01:57:46,739 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T01:57:46,744 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:46,745 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732154296745 2024-11-21T01:57:46,747 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T01:57:46,745 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T01:57:46,751 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T01:57:46,751 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55789, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T01:57:46,751 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43577, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T01:57:46,755 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T01:57:46,755 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T01:57:46,756 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T01:57:46,756 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T01:57:46,757 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:46,760 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35875 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:57:46,761 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T01:57:46,763 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T01:57:46,763 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T01:57:46,768 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35875 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T01:57:46,769 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T01:57:46,770 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T01:57:46,772 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.large.0-1732154266771,5,FailOnTimeoutGroup] 2024-11-21T01:57:46,779 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.small.0-1732154266772,5,FailOnTimeoutGroup] 2024-11-21T01:57:46,780 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:46,780 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T01:57:46,781 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:46,782 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:46,788 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T01:57:46,789 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060 2024-11-21T01:57:46,794 DEBUG [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-21T01:57:46,794 DEBUG [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-21T01:57:46,794 WARN [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-21T01:57:46,794 WARN [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-21T01:57:46,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:57:46,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T01:57:46,805 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T01:57:46,806 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:46,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:46,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T01:57:46,810 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T01:57:46,811 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:46,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:46,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T01:57:46,817 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T01:57:46,818 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:46,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:46,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T01:57:46,822 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T01:57:46,822 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:46,822 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:46,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T01:57:46,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740 2024-11-21T01:57:46,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740 2024-11-21T01:57:46,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T01:57:46,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T01:57:46,825 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T01:57:46,825 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T01:57:46,832 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T01:57:46,833 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75029501, jitterRate=0.11802668869495392}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T01:57:46,836 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732154266801Initializing all the Stores at 1732154266801Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154266801Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154266801Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732154266801Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154266801Cleaning up temporary data from old regions at 1732154266824 (+23 ms)Region opened successfully at 1732154266836 (+12 ms) 2024-11-21T01:57:46,837 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T01:57:46,837 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T01:57:46,837 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T01:57:46,838 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T01:57:46,838 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T01:57:46,839 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T01:57:46,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732154266837Disabling compacts and flushes for region at 1732154266837Disabling writes for close at 1732154266838 (+1 ms)Writing region close event to WAL at 1732154266839 (+1 ms)Closed at 1732154266839 2024-11-21T01:57:46,842 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T01:57:46,843 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T01:57:46,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T01:57:46,858 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T01:57:46,860 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T01:57:46,895 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(2659): reportForDuty to master=b93ada835ef8,35875,1732154264631 with port=37077, startcode=1732154265666 2024-11-21T01:57:46,895 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(2659): reportForDuty to master=b93ada835ef8,35875,1732154264631 with port=36587, startcode=1732154265478 2024-11-21T01:57:46,898 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35875 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b93ada835ef8,37077,1732154265666 2024-11-21T01:57:46,901 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35875 {}] master.ServerManager(517): Registering regionserver=b93ada835ef8,37077,1732154265666 2024-11-21T01:57:46,908 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35875 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b93ada835ef8,36587,1732154265478 2024-11-21T01:57:46,908 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35875 {}] master.ServerManager(517): Registering regionserver=b93ada835ef8,36587,1732154265478 2024-11-21T01:57:46,909 DEBUG [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060 2024-11-21T01:57:46,909 DEBUG [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T01:57:46,909 DEBUG [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T01:57:46,910 DEBUG [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060 2024-11-21T01:57:46,910 DEBUG [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T01:57:46,911 DEBUG [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T01:57:46,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T01:57:46,920 DEBUG [RS:0;b93ada835ef8:36587 {}] zookeeper.ZKUtil(111): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b93ada835ef8,36587,1732154265478 2024-11-21T01:57:46,920 WARN [RS:0;b93ada835ef8:36587 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T01:57:46,920 INFO [RS:0;b93ada835ef8:36587 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T01:57:46,920 DEBUG [RS:1;b93ada835ef8:37077 {}] zookeeper.ZKUtil(111): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b93ada835ef8,37077,1732154265666 2024-11-21T01:57:46,921 WARN [RS:1;b93ada835ef8:37077 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T01:57:46,921 DEBUG [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,36587,1732154265478 2024-11-21T01:57:46,922 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b93ada835ef8,36587,1732154265478] 2024-11-21T01:57:46,923 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b93ada835ef8,37077,1732154265666] 2024-11-21T01:57:46,923 INFO [RS:1;b93ada835ef8:37077 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T01:57:46,923 DEBUG [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,37077,1732154265666 2024-11-21T01:57:46,956 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T01:57:46,959 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T01:57:46,977 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T01:57:46,978 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T01:57:46,984 INFO [RS:1;b93ada835ef8:37077 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T01:57:46,984 INFO [RS:0;b93ada835ef8:36587 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T01:57:46,984 INFO [RS:1;b93ada835ef8:37077 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:46,984 INFO [RS:0;b93ada835ef8:36587 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:46,986 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T01:57:46,986 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T01:57:46,991 INFO [RS:1;b93ada835ef8:37077 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T01:57:46,991 INFO [RS:0;b93ada835ef8:36587 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T01:57:46,993 INFO [RS:0;b93ada835ef8:36587 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:46,993 INFO [RS:1;b93ada835ef8:37077 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:46,993 DEBUG [RS:0;b93ada835ef8:36587 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,993 DEBUG [RS:1;b93ada835ef8:37077 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,994 DEBUG [RS:0;b93ada835ef8:36587 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,994 DEBUG [RS:1;b93ada835ef8:37077 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,994 DEBUG [RS:0;b93ada835ef8:36587 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,994 DEBUG [RS:1;b93ada835ef8:37077 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,994 DEBUG [RS:0;b93ada835ef8:36587 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,994 DEBUG [RS:1;b93ada835ef8:37077 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,994 DEBUG [RS:0;b93ada835ef8:36587 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,994 DEBUG [RS:1;b93ada835ef8:37077 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,994 DEBUG [RS:1;b93ada835ef8:37077 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b93ada835ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-21T01:57:46,994 DEBUG [RS:0;b93ada835ef8:36587 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b93ada835ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-21T01:57:46,995 DEBUG [RS:1;b93ada835ef8:37077 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,995 DEBUG [RS:0;b93ada835ef8:36587 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,995 DEBUG [RS:0;b93ada835ef8:36587 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,995 DEBUG [RS:1;b93ada835ef8:37077 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,995 DEBUG [RS:0;b93ada835ef8:36587 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,995 DEBUG [RS:1;b93ada835ef8:37077 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,995 DEBUG [RS:1;b93ada835ef8:37077 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,996 DEBUG [RS:0;b93ada835ef8:36587 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,996 DEBUG [RS:1;b93ada835ef8:37077 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,996 DEBUG [RS:0;b93ada835ef8:36587 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,996 DEBUG [RS:1;b93ada835ef8:37077 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,996 DEBUG [RS:0;b93ada835ef8:36587 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:46,996 DEBUG [RS:1;b93ada835ef8:37077 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b93ada835ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-21T01:57:46,996 DEBUG [RS:0;b93ada835ef8:36587 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b93ada835ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-21T01:57:46,996 DEBUG [RS:1;b93ada835ef8:37077 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-21T01:57:46,997 DEBUG [RS:0;b93ada835ef8:36587 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-21T01:57:46,998 INFO [RS:1;b93ada835ef8:37077 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:46,998 INFO [RS:1;b93ada835ef8:37077 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:46,998 INFO [RS:1;b93ada835ef8:37077 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:46,998 INFO [RS:1;b93ada835ef8:37077 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:46,998 INFO [RS:1;b93ada835ef8:37077 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:46,998 INFO [RS:1;b93ada835ef8:37077 {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,37077,1732154265666-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T01:57:47,005 INFO [RS:0;b93ada835ef8:36587 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,005 INFO [RS:0;b93ada835ef8:36587 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,005 INFO [RS:0;b93ada835ef8:36587 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,005 INFO [RS:0;b93ada835ef8:36587 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,006 INFO [RS:0;b93ada835ef8:36587 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,006 INFO [RS:0;b93ada835ef8:36587 {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,36587,1732154265478-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T01:57:47,011 WARN [b93ada835ef8:35875 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T01:57:47,028 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T01:57:47,029 INFO [RS:1;b93ada835ef8:37077 {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,37077,1732154265666-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,030 INFO [RS:1;b93ada835ef8:37077 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,030 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.Replication(171): b93ada835ef8,37077,1732154265666 started 2024-11-21T01:57:47,034 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T01:57:47,035 INFO [RS:0;b93ada835ef8:36587 {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,36587,1732154265478-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,035 INFO [RS:0;b93ada835ef8:36587 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,035 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.Replication(171): b93ada835ef8,36587,1732154265478 started 2024-11-21T01:57:47,050 INFO [RS:1;b93ada835ef8:37077 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,051 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(1482): Serving as b93ada835ef8,37077,1732154265666, RpcServer on b93ada835ef8/172.17.0.3:37077, sessionid=0x10113037fa70002 2024-11-21T01:57:47,052 DEBUG [RS:1;b93ada835ef8:37077 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T01:57:47,052 DEBUG [RS:1;b93ada835ef8:37077 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b93ada835ef8,37077,1732154265666 2024-11-21T01:57:47,052 DEBUG [RS:1;b93ada835ef8:37077 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b93ada835ef8,37077,1732154265666' 2024-11-21T01:57:47,052 DEBUG [RS:1;b93ada835ef8:37077 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T01:57:47,055 DEBUG [RS:1;b93ada835ef8:37077 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T01:57:47,055 DEBUG [RS:1;b93ada835ef8:37077 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T01:57:47,055 DEBUG [RS:1;b93ada835ef8:37077 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T01:57:47,055 DEBUG [RS:1;b93ada835ef8:37077 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b93ada835ef8,37077,1732154265666 2024-11-21T01:57:47,056 DEBUG [RS:1;b93ada835ef8:37077 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b93ada835ef8,37077,1732154265666' 2024-11-21T01:57:47,056 DEBUG [RS:1;b93ada835ef8:37077 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T01:57:47,056 DEBUG [RS:1;b93ada835ef8:37077 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T01:57:47,057 DEBUG [RS:1;b93ada835ef8:37077 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T01:57:47,057 INFO [RS:1;b93ada835ef8:37077 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T01:57:47,057 INFO [RS:1;b93ada835ef8:37077 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T01:57:47,059 INFO [RS:0;b93ada835ef8:36587 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,060 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(1482): Serving as b93ada835ef8,36587,1732154265478, RpcServer on b93ada835ef8/172.17.0.3:36587, sessionid=0x10113037fa70001 2024-11-21T01:57:47,060 DEBUG [RS:0;b93ada835ef8:36587 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T01:57:47,060 DEBUG [RS:0;b93ada835ef8:36587 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b93ada835ef8,36587,1732154265478 2024-11-21T01:57:47,060 DEBUG [RS:0;b93ada835ef8:36587 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b93ada835ef8,36587,1732154265478' 2024-11-21T01:57:47,060 DEBUG [RS:0;b93ada835ef8:36587 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T01:57:47,062 DEBUG [RS:0;b93ada835ef8:36587 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T01:57:47,063 DEBUG [RS:0;b93ada835ef8:36587 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T01:57:47,063 DEBUG [RS:0;b93ada835ef8:36587 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T01:57:47,063 DEBUG [RS:0;b93ada835ef8:36587 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b93ada835ef8,36587,1732154265478 2024-11-21T01:57:47,063 DEBUG [RS:0;b93ada835ef8:36587 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b93ada835ef8,36587,1732154265478' 2024-11-21T01:57:47,063 DEBUG [RS:0;b93ada835ef8:36587 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T01:57:47,064 DEBUG [RS:0;b93ada835ef8:36587 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T01:57:47,066 DEBUG [RS:0;b93ada835ef8:36587 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T01:57:47,066 INFO [RS:0;b93ada835ef8:36587 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T01:57:47,066 INFO [RS:0;b93ada835ef8:36587 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T01:57:47,163 INFO [RS:1;b93ada835ef8:37077 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T01:57:47,164 INFO [RS:1;b93ada835ef8:37077 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=b93ada835ef8%2C37077%2C1732154265666, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,37077,1732154265666, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/oldWALs, maxLogs=55 2024-11-21T01:57:47,167 INFO [RS:0;b93ada835ef8:36587 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T01:57:47,168 INFO [RS:0;b93ada835ef8:36587 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=b93ada835ef8%2C36587%2C1732154265478, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,36587,1732154265478, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/oldWALs, maxLogs=55 2024-11-21T01:57:47,186 INFO [RS:1;b93ada835ef8:37077 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,37077,1732154265666/b93ada835ef8%2C37077%2C1732154265666.1732154267166 2024-11-21T01:57:47,187 DEBUG [RS:1;b93ada835ef8:37077 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T01:57:47,188 INFO [RS:0;b93ada835ef8:36587 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,36587,1732154265478/b93ada835ef8%2C36587%2C1732154265478.1732154267170 2024-11-21T01:57:47,189 DEBUG [RS:0;b93ada835ef8:36587 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T01:57:47,264 DEBUG [b93ada835ef8:35875 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-21T01:57:47,279 DEBUG [b93ada835ef8:35875 {}] balancer.BalancerClusterState(204): Hosts are {b93ada835ef8=0} racks are {/default-rack=0} 2024-11-21T01:57:47,282 DEBUG [b93ada835ef8:35875 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T01:57:47,282 DEBUG [b93ada835ef8:35875 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T01:57:47,282 DEBUG [b93ada835ef8:35875 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T01:57:47,282 DEBUG [b93ada835ef8:35875 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T01:57:47,282 INFO [b93ada835ef8:35875 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T01:57:47,282 INFO [b93ada835ef8:35875 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T01:57:47,282 DEBUG [b93ada835ef8:35875 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T01:57:47,290 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b93ada835ef8,37077,1732154265666 2024-11-21T01:57:47,296 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b93ada835ef8,37077,1732154265666, state=OPENING 2024-11-21T01:57:47,302 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T01:57:47,304 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:47,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:47,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:47,305 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:47,305 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:47,305 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:47,307 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T01:57:47,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b93ada835ef8,37077,1732154265666}] 2024-11-21T01:57:47,485 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T01:57:47,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54619, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T01:57:47,498 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T01:57:47,498 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T01:57:47,499 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T01:57:47,500 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=b93ada835ef8%2C37077%2C1732154265666.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,37077,1732154265666, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/oldWALs, maxLogs=55 2024-11-21T01:57:47,532 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,37077,1732154265666/b93ada835ef8%2C37077%2C1732154265666.meta.1732154267502.meta 2024-11-21T01:57:47,532 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T01:57:47,533 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:57:47,535 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T01:57:47,539 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T01:57:47,545 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T01:57:47,551 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T01:57:47,552 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:57:47,552 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T01:57:47,552 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T01:57:47,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T01:57:47,555 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T01:57:47,556 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:47,556 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:47,556 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T01:57:47,557 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T01:57:47,557 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:47,558 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:47,558 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T01:57:47,559 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T01:57:47,559 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:47,559 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:47,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T01:57:47,560 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T01:57:47,560 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:47,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:47,561 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T01:57:47,562 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740 2024-11-21T01:57:47,563 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740 2024-11-21T01:57:47,564 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T01:57:47,564 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T01:57:47,564 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T01:57:47,565 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T01:57:47,566 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59366054, jitterRate=-0.11537685990333557}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T01:57:47,566 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T01:57:47,568 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732154267553Writing region info on filesystem at 1732154267553Initializing all the Stores at 1732154267553Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154267554 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154267554Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732154267554Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154267554Cleaning up temporary data from old regions at 1732154267564 (+10 ms)Running coprocessor post-open hooks at 1732154267566 (+2 ms)Region opened successfully at 1732154267568 (+2 ms) 2024-11-21T01:57:47,575 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732154267475 2024-11-21T01:57:47,585 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T01:57:47,586 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T01:57:47,587 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b93ada835ef8,37077,1732154265666 2024-11-21T01:57:47,588 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b93ada835ef8,37077,1732154265666, state=OPEN 2024-11-21T01:57:47,596 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T01:57:47,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T01:57:47,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T01:57:47,596 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:47,596 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:47,596 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:47,597 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b93ada835ef8,37077,1732154265666 2024-11-21T01:57:47,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T01:57:47,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b93ada835ef8,37077,1732154265666 in 288 msec 2024-11-21T01:57:47,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T01:57:47,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 753 msec 2024-11-21T01:57:47,607 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T01:57:47,607 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T01:57:47,631 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:57:47,632 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,37077,1732154265666, seqNum=-1] 2024-11-21T01:57:47,656 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:57:47,658 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39073, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:57:47,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0600 sec 2024-11-21T01:57:47,678 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732154267678, completionTime=-1 2024-11-21T01:57:47,681 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-21T01:57:47,682 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T01:57:47,715 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T01:57:47,715 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732154327715 2024-11-21T01:57:47,716 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732154387716 2024-11-21T01:57:47,716 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 34 msec 2024-11-21T01:57:47,719 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-21T01:57:47,727 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,35875,1732154264631-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,728 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,35875,1732154264631-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,728 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,35875,1732154264631-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,730 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b93ada835ef8:35875, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,730 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,731 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,738 DEBUG [master/b93ada835ef8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T01:57:47,776 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.980sec 2024-11-21T01:57:47,779 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T01:57:47,780 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T01:57:47,782 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T01:57:47,782 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T01:57:47,782 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T01:57:47,783 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,35875,1732154264631-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T01:57:47,784 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,35875,1732154264631-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T01:57:47,786 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T01:57:47,787 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T01:57:47,787 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,35875,1732154264631-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:47,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109a9e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:57:47,839 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T01:57:47,839 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T01:57:47,850 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,35875,-1 for getting cluster id 2024-11-21T01:57:47,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:57:47,864 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd5d4c01c-9937-4836-9676-fc2e82ab4603' 2024-11-21T01:57:47,867 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:57:47,867 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d5d4c01c-9937-4836-9676-fc2e82ab4603" 2024-11-21T01:57:47,868 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a62a8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:57:47,868 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,35875,-1] 2024-11-21T01:57:47,872 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:57:47,877 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:47,878 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:57:47,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36c35605, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:57:47,884 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:57:47,893 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,37077,1732154265666, seqNum=-1] 2024-11-21T01:57:47,894 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:57:47,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:57:47,931 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b93ada835ef8,35875,1732154264631 2024-11-21T01:57:47,935 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T01:57:47,943 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b93ada835ef8,35875,1732154264631 2024-11-21T01:57:47,946 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f886de7 2024-11-21T01:57:47,946 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T01:57:47,949 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T01:57:47,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35875 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testBlockEvictionOnRegionMove1732154267931', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T01:57:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35875 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732154267931 2024-11-21T01:57:47,963 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732154267931 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T01:57:47,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35875 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732154267931" procId is: 4 2024-11-21T01:57:47,966 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:47,967 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732154267931 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T01:57:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35875 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T01:57:47,982 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732154267931-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 304eabe0c3e5c72e137fb13f7787c56a, NAME => 'testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732154267931', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060 2024-11-21T01:57:47,989 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732154267931-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:57:47,989 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732154267931-pool-0 {}] regionserver.HRegion(1722): Closing 304eabe0c3e5c72e137fb13f7787c56a, disabling compactions & flushes 2024-11-21T01:57:47,989 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732154267931-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:47,989 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732154267931-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:47,989 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732154267931-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. after waiting 0 ms 2024-11-21T01:57:47,989 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732154267931-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:47,989 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732154267931-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:47,989 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732154267931-pool-0 {}] regionserver.HRegion(1676): Region close journal for 304eabe0c3e5c72e137fb13f7787c56a: Waiting for close lock at 1732154267989Disabling compacts and flushes for region at 1732154267989Disabling writes for close at 1732154267989Writing region close event to WAL at 1732154267989Closed at 1732154267989 2024-11-21T01:57:47,990 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732154267931 execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T01:57:47,995 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732154267991"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732154267991"}]},"ts":"1732154267991"} 2024-11-21T01:57:48,000 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T01:57:48,001 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732154267931 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T01:57:48,003 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732154267931","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732154268001"}]},"ts":"1732154268001"} 2024-11-21T01:57:48,007 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732154267931, state=ENABLING in hbase:meta 2024-11-21T01:57:48,008 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {b93ada835ef8=0} racks are {/default-rack=0} 2024-11-21T01:57:48,008 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T01:57:48,008 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T01:57:48,008 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T01:57:48,008 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T01:57:48,008 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T01:57:48,008 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T01:57:48,008 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T01:57:48,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732154267931, region=304eabe0c3e5c72e137fb13f7787c56a, ASSIGN}] 2024-11-21T01:57:48,011 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732154267931, region=304eabe0c3e5c72e137fb13f7787c56a, ASSIGN 2024-11-21T01:57:48,012 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732154267931, region=304eabe0c3e5c72e137fb13f7787c56a, ASSIGN; state=OFFLINE, location=b93ada835ef8,37077,1732154265666; forceNewPlan=false, retain=false 2024-11-21T01:57:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35875 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T01:57:48,165 INFO [b93ada835ef8:35875 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T01:57:48,166 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=304eabe0c3e5c72e137fb13f7787c56a, regionState=OPENING, regionLocation=b93ada835ef8,37077,1732154265666 2024-11-21T01:57:48,169 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732154267931, region=304eabe0c3e5c72e137fb13f7787c56a, ASSIGN because future has completed 2024-11-21T01:57:48,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 304eabe0c3e5c72e137fb13f7787c56a, server=b93ada835ef8,37077,1732154265666}] 2024-11-21T01:57:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35875 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T01:57:48,327 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:48,328 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 304eabe0c3e5c72e137fb13f7787c56a, NAME => 'testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:57:48,328 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732154267931 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:48,328 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:57:48,329 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:48,329 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:48,330 INFO [StoreOpener-304eabe0c3e5c72e137fb13f7787c56a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:48,337 INFO [StoreOpener-304eabe0c3e5c72e137fb13f7787c56a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 304eabe0c3e5c72e137fb13f7787c56a columnFamilyName family 2024-11-21T01:57:48,337 DEBUG [StoreOpener-304eabe0c3e5c72e137fb13f7787c56a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:48,338 INFO [StoreOpener-304eabe0c3e5c72e137fb13f7787c56a-1 {}] regionserver.HStore(327): Store=304eabe0c3e5c72e137fb13f7787c56a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:57:48,338 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:48,340 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:48,340 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:48,341 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:48,341 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:48,342 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:48,350 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T01:57:48,351 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 304eabe0c3e5c72e137fb13f7787c56a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64000321, jitterRate=-0.046320900321006775}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T01:57:48,351 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:48,352 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 304eabe0c3e5c72e137fb13f7787c56a: Running coprocessor pre-open hook at 1732154268329Writing region info on filesystem at 1732154268329Initializing all the Stores at 1732154268329Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732154268329Cleaning up temporary data from old regions at 1732154268341 (+12 ms)Running coprocessor post-open hooks at 1732154268351 (+10 ms)Region opened successfully at 1732154268352 (+1 ms) 2024-11-21T01:57:48,354 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a., pid=6, masterSystemTime=1732154268323 2024-11-21T01:57:48,358 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:48,358 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:48,358 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=304eabe0c3e5c72e137fb13f7787c56a, regionState=OPEN, openSeqNum=2, regionLocation=b93ada835ef8,37077,1732154265666 2024-11-21T01:57:48,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 304eabe0c3e5c72e137fb13f7787c56a, server=b93ada835ef8,37077,1732154265666 because future has completed 2024-11-21T01:57:48,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T01:57:48,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 304eabe0c3e5c72e137fb13f7787c56a, server=b93ada835ef8,37077,1732154265666 in 196 msec 2024-11-21T01:57:48,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T01:57:48,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732154267931, region=304eabe0c3e5c72e137fb13f7787c56a, ASSIGN in 359 msec 2024-11-21T01:57:48,373 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732154267931 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T01:57:48,373 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732154267931","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732154268373"}]},"ts":"1732154268373"} 2024-11-21T01:57:48,376 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732154267931, state=ENABLED in hbase:meta 2024-11-21T01:57:48,377 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732154267931 execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T01:57:48,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732154267931 in 420 msec 2024-11-21T01:57:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35875 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T01:57:48,601 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732154267931 get assigned. Timeout = 60000ms 2024-11-21T01:57:48,601 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732154267931 completed 2024-11-21T01:57:48,605 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T01:57:48,610 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732154267931 assigned to meta. Checking AM states. 2024-11-21T01:57:48,611 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T01:57:48,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732154267931 assigned. 2024-11-21T01:57:48,619 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732154267931', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a., hostname=b93ada835ef8,37077,1732154265666, seqNum=2] 2024-11-21T01:57:48,631 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 304eabe0c3e5c72e137fb13f7787c56a 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-21T01:57:48,660 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f0cfcc4116f0492d9c06e375aa8472d8 2024-11-21T01:57:48,698 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/.tmp/family/f0cfcc4116f0492d9c06e375aa8472d8 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-21T01:57:48,704 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/.tmp/family/f0cfcc4116f0492d9c06e375aa8472d8 2024-11-21T01:57:48,738 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/.tmp/family/f0cfcc4116f0492d9c06e375aa8472d8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/family/f0cfcc4116f0492d9c06e375aa8472d8 2024-11-21T01:57:48,740 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/family/f0cfcc4116f0492d9c06e375aa8472d8, delay=949 ms 2024-11-21T01:57:48,741 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/family/f0cfcc4116f0492d9c06e375aa8472d8, entries=2, sequenceid=6, filesize=4.9 K 2024-11-21T01:57:48,748 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 304eabe0c3e5c72e137fb13f7787c56a in 116ms, sequenceid=6, compaction requested=false 2024-11-21T01:57:48,749 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732154267931' 2024-11-21T01:57:48,751 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 304eabe0c3e5c72e137fb13f7787c56a: 2024-11-21T01:57:49,691 DEBUG [hfile-prefetch-1732154268740 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f0cfcc4116f0492d9c06e375aa8472d8_0. 2024-11-21T01:57:49,695 DEBUG [hfile-prefetch-1732154268740 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f0cfcc4116f0492d9c06e375aa8472d8_117. 2024-11-21T01:57:49,696 DEBUG [hfile-prefetch-1732154268740 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/family/f0cfcc4116f0492d9c06e375aa8472d8, with total blocks 2, and data blocks 1 2024-11-21T01:57:49,696 DEBUG [hfile-prefetch-1732154268740 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-21T01:57:49,697 DEBUG [hfile-prefetch-1732154268740 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/family/f0cfcc4116f0492d9c06e375aa8472d8, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-21T01:57:49,697 DEBUG [hfile-prefetch-1732154268740 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/family/f0cfcc4116f0492d9c06e375aa8472d8. Will sleep 100ms and try the verification again. 2024-11-21T01:57:49,701 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T01:57:49,701 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T01:57:49,702 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T01:57:49,702 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T01:57:49,727 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-21T01:57:49,727 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732154269702 2024-11-21T01:57:49,727 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T01:57:49,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35875 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=304eabe0c3e5c72e137fb13f7787c56a, source=b93ada835ef8,37077,1732154265666, destination=b93ada835ef8,36587,1732154265478, warming up region on b93ada835ef8,36587,1732154265478 2024-11-21T01:57:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35875 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T01:57:49,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35875 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=304eabe0c3e5c72e137fb13f7787c56a, source=b93ada835ef8,37077,1732154265666, destination=b93ada835ef8,36587,1732154265478, running balancer 2024-11-21T01:57:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35875 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732154267931, region=304eabe0c3e5c72e137fb13f7787c56a, REOPEN/MOVE 2024-11-21T01:57:49,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48691, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T01:57:49,766 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732154267931, region=304eabe0c3e5c72e137fb13f7787c56a, REOPEN/MOVE 2024-11-21T01:57:49,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36587 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36587 {}] regionserver.HRegion(7855): Warmup {ENCODED => 304eabe0c3e5c72e137fb13f7787c56a, NAME => 'testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:57:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36587 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:57:49,768 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=304eabe0c3e5c72e137fb13f7787c56a, regionState=CLOSING, regionLocation=b93ada835ef8,37077,1732154265666 2024-11-21T01:57:49,769 INFO [StoreOpener-304eabe0c3e5c72e137fb13f7787c56a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:49,770 INFO [StoreOpener-304eabe0c3e5c72e137fb13f7787c56a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 304eabe0c3e5c72e137fb13f7787c56a columnFamilyName family 2024-11-21T01:57:49,770 DEBUG [StoreOpener-304eabe0c3e5c72e137fb13f7787c56a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:49,772 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732154267931, region=304eabe0c3e5c72e137fb13f7787c56a, REOPEN/MOVE because future has completed 2024-11-21T01:57:49,773 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-21T01:57:49,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 304eabe0c3e5c72e137fb13f7787c56a, server=b93ada835ef8,37077,1732154265666}] 2024-11-21T01:57:49,774 DEBUG [StoreOpener-304eabe0c3e5c72e137fb13f7787c56a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/family/f0cfcc4116f0492d9c06e375aa8472d8 2024-11-21T01:57:49,774 INFO [StoreOpener-304eabe0c3e5c72e137fb13f7787c56a-1 {}] regionserver.HStore(327): Store=304eabe0c3e5c72e137fb13f7787c56a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:57:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36587 {}] regionserver.HRegion(1722): Closing 304eabe0c3e5c72e137fb13f7787c56a, disabling compactions & flushes 2024-11-21T01:57:49,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36587 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36587 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36587 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. after waiting 0 ms 2024-11-21T01:57:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36587 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:49,776 DEBUG [StoreFileCloser-304eabe0c3e5c72e137fb13f7787c56a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/family/f0cfcc4116f0492d9c06e375aa8472d8 2024-11-21T01:57:49,776 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36587 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36587 {}] regionserver.HRegion(1676): Region close journal for 304eabe0c3e5c72e137fb13f7787c56a: Waiting for close lock at 1732154269774Disabling compacts and flushes for region at 1732154269774Disabling writes for close at 1732154269774Writing region close event to WAL at 1732154269776 (+2 ms)Closed at 1732154269776 2024-11-21T01:57:49,778 WARN [hfile-prefetch-1732154268740 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/family/f0cfcc4116f0492d9c06e375aa8472d8, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-21T01:57:49,779 DEBUG [hfile-prefetch-1732154268740 {}] hfile.PrefetchExecutor(121): Prefetch completed for f0cfcc4116f0492d9c06e375aa8472d8 2024-11-21T01:57:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35875 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-21T01:57:49,931 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:49,931 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-21T01:57:49,932 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 304eabe0c3e5c72e137fb13f7787c56a, disabling compactions & flushes 2024-11-21T01:57:49,932 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:49,932 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:49,932 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. after waiting 0 ms 2024-11-21T01:57:49,932 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:49,934 DEBUG [StoreFileCloser-304eabe0c3e5c72e137fb13f7787c56a-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file f0cfcc4116f0492d9c06e375aa8472d8, starting offset: 0, end offset: 9223372036854775807 2024-11-21T01:57:49,936 DEBUG [StoreFileCloser-304eabe0c3e5c72e137fb13f7787c56a-family-1 {}] bucket.BucketCache(856): removed key f0cfcc4116f0492d9c06e375aa8472d8_0 from back map with offset lock 2094080 in the evict process 2024-11-21T01:57:49,942 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-21T01:57:49,944 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:49,944 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 304eabe0c3e5c72e137fb13f7787c56a: Waiting for close lock at 1732154269932Running coprocessor pre-close hooks at 1732154269932Disabling compacts and flushes for region at 1732154269932Disabling writes for close at 1732154269932Writing region close event to WAL at 1732154269936 (+4 ms)Running coprocessor post-close hooks at 1732154269943 (+7 ms)Closed at 1732154269944 (+1 ms) 2024-11-21T01:57:49,945 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 304eabe0c3e5c72e137fb13f7787c56a move to b93ada835ef8,36587,1732154265478 record at close sequenceid=6 2024-11-21T01:57:49,948 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:49,949 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=304eabe0c3e5c72e137fb13f7787c56a, regionState=CLOSED 2024-11-21T01:57:49,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 304eabe0c3e5c72e137fb13f7787c56a, server=b93ada835ef8,37077,1732154265666 because future has completed 2024-11-21T01:57:49,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T01:57:49,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 304eabe0c3e5c72e137fb13f7787c56a, server=b93ada835ef8,37077,1732154265666 in 180 msec 2024-11-21T01:57:49,956 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732154267931, region=304eabe0c3e5c72e137fb13f7787c56a, REOPEN/MOVE; state=CLOSED, location=b93ada835ef8,36587,1732154265478; forceNewPlan=false, retain=false 2024-11-21T01:57:49,990 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T01:57:49,990 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T01:57:49,990 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T01:57:49,990 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T01:57:49,993 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T01:57:49,993 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732154269990 2024-11-21T01:57:49,993 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T01:57:50,106 INFO [b93ada835ef8:35875 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T01:57:50,107 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=304eabe0c3e5c72e137fb13f7787c56a, regionState=OPENING, regionLocation=b93ada835ef8,36587,1732154265478 2024-11-21T01:57:50,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732154267931, region=304eabe0c3e5c72e137fb13f7787c56a, REOPEN/MOVE because future has completed 2024-11-21T01:57:50,111 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 304eabe0c3e5c72e137fb13f7787c56a, server=b93ada835ef8,36587,1732154265478}] 2024-11-21T01:57:50,267 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:50,267 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 304eabe0c3e5c72e137fb13f7787c56a, NAME => 'testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:57:50,268 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732154267931 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:50,268 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:57:50,268 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:50,268 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:50,269 INFO [StoreOpener-304eabe0c3e5c72e137fb13f7787c56a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:50,271 INFO [StoreOpener-304eabe0c3e5c72e137fb13f7787c56a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 304eabe0c3e5c72e137fb13f7787c56a columnFamilyName family 2024-11-21T01:57:50,271 DEBUG [StoreOpener-304eabe0c3e5c72e137fb13f7787c56a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:50,273 DEBUG [StoreFileOpener-304eabe0c3e5c72e137fb13f7787c56a-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/family/f0cfcc4116f0492d9c06e375aa8472d8, delay=912 ms 2024-11-21T01:57:50,274 DEBUG [StoreOpener-304eabe0c3e5c72e137fb13f7787c56a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/family/f0cfcc4116f0492d9c06e375aa8472d8 2024-11-21T01:57:50,274 INFO [StoreOpener-304eabe0c3e5c72e137fb13f7787c56a-1 {}] regionserver.HStore(327): Store=304eabe0c3e5c72e137fb13f7787c56a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:57:50,275 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:50,276 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:50,277 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:50,277 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:50,277 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:50,282 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:50,282 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 304eabe0c3e5c72e137fb13f7787c56a; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75076328, jitterRate=0.11872446537017822}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T01:57:50,282 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:50,283 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 304eabe0c3e5c72e137fb13f7787c56a: Running coprocessor pre-open hook at 1732154270268Writing region info on filesystem at 1732154270268Initializing all the Stores at 1732154270269 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732154270269Cleaning up temporary data from old regions at 1732154270277 (+8 ms)Running coprocessor post-open hooks at 1732154270282 (+5 ms)Region opened successfully at 1732154270283 (+1 ms) 2024-11-21T01:57:50,284 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a., pid=9, masterSystemTime=1732154270263 2024-11-21T01:57:50,287 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:50,287 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:50,287 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=304eabe0c3e5c72e137fb13f7787c56a, regionState=OPEN, openSeqNum=10, regionLocation=b93ada835ef8,36587,1732154265478 2024-11-21T01:57:50,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 304eabe0c3e5c72e137fb13f7787c56a, server=b93ada835ef8,36587,1732154265478 because future has completed 2024-11-21T01:57:50,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-21T01:57:50,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 304eabe0c3e5c72e137fb13f7787c56a, server=b93ada835ef8,36587,1732154265478 in 181 msec 2024-11-21T01:57:50,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732154267931, region=304eabe0c3e5c72e137fb13f7787c56a, REOPEN/MOVE in 529 msec 2024-11-21T01:57:50,318 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T01:57:50,319 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T01:57:50,319 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:57:50,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:50,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:50,324 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:57:50,324 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T01:57:50,324 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-21T01:57:50,324 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b93ada835ef8,35875,1732154264631 2024-11-21T01:57:50,326 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T01:57:50,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T01:57:50,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T01:57:50,327 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:50,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:50,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:50,327 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T01:57:50,328 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T01:57:50,328 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T01:57:50,328 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T01:57:50,328 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T01:57:50,328 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:57:50,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:50,329 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b93ada835ef8,36587,1732154265478' ***** 2024-11-21T01:57:50,329 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T01:57:50,329 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b93ada835ef8,37077,1732154265666' ***** 2024-11-21T01:57:50,329 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T01:57:50,329 INFO [RS:0;b93ada835ef8:36587 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T01:57:50,329 INFO [RS:1;b93ada835ef8:37077 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T01:57:50,330 INFO [RS:1;b93ada835ef8:37077 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/bucket.persistence 2024-11-21T01:57:50,330 INFO [RS:0;b93ada835ef8:36587 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/bucket.persistence 2024-11-21T01:57:50,330 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T01:57:50,330 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T01:57:50,330 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T01:57:50,330 INFO [RS:0;b93ada835ef8:36587 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T01:57:50,330 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T01:57:50,330 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T01:57:50,330 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T01:57:50,330 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T01:57:50,330 INFO [RS:1;b93ada835ef8:37077 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T01:57:50,330 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T01:57:50,340 DEBUG [RS:0;b93ada835ef8:36587 {}] bucket.BucketCache(1487): Thread RS:0;b93ada835ef8:36587 started persisting bucket cache to file 2024-11-21T01:57:50,340 DEBUG [RS:0;b93ada835ef8:36587 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T01:57:50,340 DEBUG [RS:0;b93ada835ef8:36587 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T01:57:50,340 DEBUG [RS:1;b93ada835ef8:37077 {}] bucket.BucketCache(1487): Thread RS:1;b93ada835ef8:37077 started persisting bucket cache to file 2024-11-21T01:57:50,341 DEBUG [RS:1;b93ada835ef8:37077 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T01:57:50,341 DEBUG [RS:1;b93ada835ef8:37077 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T01:57:50,344 DEBUG [RS:0;b93ada835ef8:36587 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T01:57:50,344 DEBUG [RS:0;b93ada835ef8:36587 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732154270340 2024-11-21T01:57:50,344 DEBUG [RS:0;b93ada835ef8:36587 {}] bucket.BucketCache(1509): Thread RS:0;b93ada835ef8:36587 finished persisting bucket cache to file, renaming 2024-11-21T01:57:50,344 DEBUG [RS:1;b93ada835ef8:37077 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T01:57:50,344 DEBUG [RS:1;b93ada835ef8:37077 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732154270341 2024-11-21T01:57:50,344 DEBUG [RS:1;b93ada835ef8:37077 {}] bucket.BucketCache(1509): Thread RS:1;b93ada835ef8:37077 finished persisting bucket cache to file, renaming 2024-11-21T01:57:50,344 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T01:57:50,345 INFO [RS:0;b93ada835ef8:36587 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T01:57:50,345 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T01:57:50,345 INFO [RS:0;b93ada835ef8:36587 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T01:57:50,345 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(3091): Received CLOSE for 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:50,345 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T01:57:50,345 INFO [RS:1;b93ada835ef8:37077 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T01:57:50,345 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T01:57:50,345 INFO [RS:1;b93ada835ef8:37077 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T01:57:50,345 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(959): stopping server b93ada835ef8,37077,1732154265666 2024-11-21T01:57:50,345 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(959): stopping server b93ada835ef8,36587,1732154265478 2024-11-21T01:57:50,345 INFO [RS:0;b93ada835ef8:36587 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T01:57:50,345 INFO [RS:1;b93ada835ef8:37077 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T01:57:50,346 INFO [RS:0;b93ada835ef8:36587 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b93ada835ef8:36587. 2024-11-21T01:57:50,346 INFO [RS:1;b93ada835ef8:37077 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;b93ada835ef8:37077. 2024-11-21T01:57:50,346 DEBUG [RS:0;b93ada835ef8:36587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:57:50,346 DEBUG [RS:0;b93ada835ef8:36587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:50,346 DEBUG [RS:1;b93ada835ef8:37077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:57:50,346 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T01:57:50,346 DEBUG [RS:1;b93ada835ef8:37077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:50,346 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-21T01:57:50,346 DEBUG [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(1325): Online Regions={304eabe0c3e5c72e137fb13f7787c56a=testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a.} 2024-11-21T01:57:50,346 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T01:57:50,346 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 304eabe0c3e5c72e137fb13f7787c56a, disabling compactions & flushes 2024-11-21T01:57:50,346 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T01:57:50,346 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:50,346 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T01:57:50,346 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:50,346 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. after waiting 0 ms 2024-11-21T01:57:50,346 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T01:57:50,346 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:50,346 DEBUG [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(1351): Waiting on 304eabe0c3e5c72e137fb13f7787c56a 2024-11-21T01:57:50,347 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-21T01:57:50,347 DEBUG [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-21T01:57:50,347 DEBUG [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T01:57:50,347 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T01:57:50,347 DEBUG [StoreFileCloser-304eabe0c3e5c72e137fb13f7787c56a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/family/f0cfcc4116f0492d9c06e375aa8472d8 2024-11-21T01:57:50,347 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T01:57:50,347 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T01:57:50,347 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T01:57:50,347 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T01:57:50,347 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T01:57:50,348 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-21T01:57:50,352 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnRegionMove1732154267931/304eabe0c3e5c72e137fb13f7787c56a/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-21T01:57:50,352 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:50,352 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 304eabe0c3e5c72e137fb13f7787c56a: Waiting for close lock at 1732154270346Running coprocessor pre-close hooks at 1732154270346Disabling compacts and flushes for region at 1732154270346Disabling writes for close at 1732154270346Writing region close event to WAL at 1732154270347 (+1 ms)Running coprocessor post-close hooks at 1732154270352 (+5 ms)Closed at 1732154270352 2024-11-21T01:57:50,353 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a. 2024-11-21T01:57:50,358 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on df2001a2c025469d954581652694ab3f 2024-11-21T01:57:50,375 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/info/df2001a2c025469d954581652694ab3f is 205, key is testBlockEvictionOnRegionMove1732154267931,,1732154267950.304eabe0c3e5c72e137fb13f7787c56a./info:regioninfo/1732154270287/Put/seqid=0 2024-11-21T01:57:50,376 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/info/df2001a2c025469d954581652694ab3f 2024-11-21T01:57:50,386 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cae2913339704faca0c455d57208d025 2024-11-21T01:57:50,400 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/ns/cae2913339704faca0c455d57208d025 is 43, key is default/ns:d/1732154267663/Put/seqid=0 2024-11-21T01:57:50,401 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/ns/cae2913339704faca0c455d57208d025 2024-11-21T01:57:50,408 INFO [regionserver/b93ada835ef8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T01:57:50,413 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b24db884a84e4b46ae3161dfa9f2cf8c 2024-11-21T01:57:50,425 INFO [regionserver/b93ada835ef8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T01:57:50,426 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/table/b24db884a84e4b46ae3161dfa9f2cf8c is 78, key is testBlockEvictionOnRegionMove1732154267931/table:state/1732154268373/Put/seqid=0 2024-11-21T01:57:50,427 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/table/b24db884a84e4b46ae3161dfa9f2cf8c 2024-11-21T01:57:50,429 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/info/df2001a2c025469d954581652694ab3f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/info/df2001a2c025469d954581652694ab3f 2024-11-21T01:57:50,430 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/info/df2001a2c025469d954581652694ab3f, delay=939 ms 2024-11-21T01:57:50,431 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/info/df2001a2c025469d954581652694ab3f, entries=15, sequenceid=15, filesize=7.6 K 2024-11-21T01:57:50,432 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/ns/cae2913339704faca0c455d57208d025 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/ns/cae2913339704faca0c455d57208d025 2024-11-21T01:57:50,433 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/ns/cae2913339704faca0c455d57208d025, delay=959 ms 2024-11-21T01:57:50,434 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/ns/cae2913339704faca0c455d57208d025, entries=2, sequenceid=15, filesize=5.0 K 2024-11-21T01:57:50,435 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/table/b24db884a84e4b46ae3161dfa9f2cf8c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/table/b24db884a84e4b46ae3161dfa9f2cf8c 2024-11-21T01:57:50,436 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/table/b24db884a84e4b46ae3161dfa9f2cf8c, delay=960 ms 2024-11-21T01:57:50,437 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/table/b24db884a84e4b46ae3161dfa9f2cf8c, entries=2, sequenceid=15, filesize=5.3 K 2024-11-21T01:57:50,438 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=15, compaction requested=false 2024-11-21T01:57:50,438 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T01:57:50,439 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/info/df2001a2c025469d954581652694ab3f 2024-11-21T01:57:50,440 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/ns/cae2913339704faca0c455d57208d025 2024-11-21T01:57:50,440 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/table/b24db884a84e4b46ae3161dfa9f2cf8c 2024-11-21T01:57:50,445 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-21T01:57:50,445 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T01:57:50,446 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T01:57:50,446 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732154270347Running coprocessor pre-close hooks at 1732154270347Disabling compacts and flushes for region at 1732154270347Disabling writes for close at 1732154270347Obtaining lock to block concurrent updates at 1732154270348 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732154270348Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732154270348Flushing stores of hbase:meta,,1.1588230740 at 1732154270348Flushing 1588230740/info: creating writer at 1732154270349 (+1 ms)Flushing 1588230740/info: appending metadata at 1732154270374 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732154270374Flushing 1588230740/ns: creating writer at 1732154270379 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732154270399 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732154270399Flushing 1588230740/table: creating writer at 1732154270404 (+5 ms)Flushing 1588230740/table: appending metadata at 1732154270426 (+22 ms)Flushing 1588230740/table: closing flushed file at 1732154270426Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d975fa3: reopening flushed file at 1732154270429 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a188d2: reopening flushed file at 1732154270432 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b1c6594: reopening flushed file at 1732154270434 (+2 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=15, compaction requested=false at 1732154270438 (+4 ms)Writing region close event to WAL at 1732154270441 (+3 ms)Running coprocessor post-close hooks at 1732154270445 (+4 ms)Closed at 1732154270446 (+1 ms) 2024-11-21T01:57:50,446 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T01:57:50,547 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(976): stopping server b93ada835ef8,36587,1732154265478; all regions closed. 2024-11-21T01:57:50,547 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(976): stopping server b93ada835ef8,37077,1732154265666; all regions closed. 2024-11-21T01:57:50,552 DEBUG [RS:1;b93ada835ef8:37077 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/oldWALs 2024-11-21T01:57:50,553 INFO [RS:1;b93ada835ef8:37077 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b93ada835ef8%2C37077%2C1732154265666.meta:.meta(num 1732154267502) 2024-11-21T01:57:50,553 DEBUG [RS:0;b93ada835ef8:36587 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/oldWALs 2024-11-21T01:57:50,553 INFO [RS:0;b93ada835ef8:36587 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b93ada835ef8%2C36587%2C1732154265478:(num 1732154267170) 2024-11-21T01:57:50,553 DEBUG [RS:0;b93ada835ef8:36587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:50,553 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T01:57:50,553 INFO [RS:0;b93ada835ef8:36587 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T01:57:50,554 INFO [RS:0;b93ada835ef8:36587 {}] hbase.ChoreService(370): Chore service for: regionserver/b93ada835ef8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T01:57:50,554 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T01:57:50,554 INFO [regionserver/b93ada835ef8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T01:57:50,554 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T01:57:50,554 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T01:57:50,554 INFO [RS:0;b93ada835ef8:36587 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T01:57:50,555 INFO [RS:0;b93ada835ef8:36587 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36587 2024-11-21T01:57:50,555 DEBUG [RS:1;b93ada835ef8:37077 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/oldWALs 2024-11-21T01:57:50,555 INFO [RS:1;b93ada835ef8:37077 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b93ada835ef8%2C37077%2C1732154265666:(num 1732154267166) 2024-11-21T01:57:50,555 DEBUG [RS:1;b93ada835ef8:37077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:50,555 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T01:57:50,556 INFO [RS:1;b93ada835ef8:37077 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T01:57:50,556 INFO [RS:1;b93ada835ef8:37077 {}] hbase.ChoreService(370): Chore service for: regionserver/b93ada835ef8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-21T01:57:50,556 INFO [RS:1;b93ada835ef8:37077 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T01:57:50,556 INFO [regionserver/b93ada835ef8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T01:57:50,556 INFO [RS:1;b93ada835ef8:37077 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37077 2024-11-21T01:57:50,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b93ada835ef8,36587,1732154265478 2024-11-21T01:57:50,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T01:57:50,561 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b93ada835ef8,37077,1732154265666 2024-11-21T01:57:50,562 INFO [RS:0;b93ada835ef8:36587 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T01:57:50,563 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b93ada835ef8,36587,1732154265478] 2024-11-21T01:57:50,565 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b93ada835ef8,36587,1732154265478 already deleted, retry=false 2024-11-21T01:57:50,565 INFO [RS:1;b93ada835ef8:37077 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T01:57:50,565 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b93ada835ef8,36587,1732154265478 expired; onlineServers=1 2024-11-21T01:57:50,565 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b93ada835ef8,37077,1732154265666] 2024-11-21T01:57:50,567 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b93ada835ef8,37077,1732154265666 already deleted, retry=false 2024-11-21T01:57:50,567 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b93ada835ef8,37077,1732154265666 expired; onlineServers=0 2024-11-21T01:57:50,567 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b93ada835ef8,35875,1732154264631' ***** 2024-11-21T01:57:50,568 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T01:57:50,568 INFO [M:0;b93ada835ef8:35875 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T01:57:50,568 INFO [M:0;b93ada835ef8:35875 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T01:57:50,568 DEBUG [M:0;b93ada835ef8:35875 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T01:57:50,568 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T01:57:50,569 DEBUG [M:0;b93ada835ef8:35875 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T01:57:50,569 DEBUG [master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.large.0-1732154266771 {}] cleaner.HFileCleaner(306): Exit Thread[master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.large.0-1732154266771,5,FailOnTimeoutGroup] 2024-11-21T01:57:50,569 INFO [M:0;b93ada835ef8:35875 {}] hbase.ChoreService(370): Chore service for: master/b93ada835ef8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T01:57:50,569 DEBUG [master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.small.0-1732154266772 {}] cleaner.HFileCleaner(306): Exit Thread[master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.small.0-1732154266772,5,FailOnTimeoutGroup] 2024-11-21T01:57:50,569 INFO [M:0;b93ada835ef8:35875 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T01:57:50,569 DEBUG [M:0;b93ada835ef8:35875 {}] master.HMaster(1795): Stopping service threads 2024-11-21T01:57:50,569 INFO [M:0;b93ada835ef8:35875 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T01:57:50,569 INFO [M:0;b93ada835ef8:35875 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T01:57:50,570 INFO [M:0;b93ada835ef8:35875 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T01:57:50,570 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T01:57:50,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T01:57:50,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:50,571 DEBUG [M:0;b93ada835ef8:35875 {}] zookeeper.ZKUtil(347): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T01:57:50,571 WARN [M:0;b93ada835ef8:35875 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T01:57:50,571 INFO [M:0;b93ada835ef8:35875 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/.lastflushedseqids 2024-11-21T01:57:50,582 INFO [M:0;b93ada835ef8:35875 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T01:57:50,583 INFO [M:0;b93ada835ef8:35875 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T01:57:50,583 DEBUG [M:0;b93ada835ef8:35875 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T01:57:50,583 INFO [M:0;b93ada835ef8:35875 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:57:50,583 DEBUG [M:0;b93ada835ef8:35875 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:57:50,583 DEBUG [M:0;b93ada835ef8:35875 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T01:57:50,583 DEBUG [M:0;b93ada835ef8:35875 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:57:50,583 INFO [M:0;b93ada835ef8:35875 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-21T01:57:50,594 DEBUG [M:0;b93ada835ef8:35875 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 731456bf625a4593ae2fe21ec09e1779 2024-11-21T01:57:50,609 DEBUG [M:0;b93ada835ef8:35875 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/731456bf625a4593ae2fe21ec09e1779 is 82, key is hbase:meta,,1/info:regioninfo/1732154267586/Put/seqid=0 2024-11-21T01:57:50,610 INFO [M:0;b93ada835ef8:35875 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/731456bf625a4593ae2fe21ec09e1779 2024-11-21T01:57:50,619 DEBUG [M:0;b93ada835ef8:35875 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3266b4bbb19c420bb0497da9c32513ed 2024-11-21T01:57:50,635 DEBUG [M:0;b93ada835ef8:35875 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3266b4bbb19c420bb0497da9c32513ed is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732154268379/Put/seqid=0 2024-11-21T01:57:50,635 INFO [M:0;b93ada835ef8:35875 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3266b4bbb19c420bb0497da9c32513ed 2024-11-21T01:57:50,643 DEBUG [M:0;b93ada835ef8:35875 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ed7da4a53b7b40c5b2f49af8abc2e2a8 2024-11-21T01:57:50,656 DEBUG [M:0;b93ada835ef8:35875 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ed7da4a53b7b40c5b2f49af8abc2e2a8 is 69, key is b93ada835ef8,36587,1732154265478/rs:state/1732154266909/Put/seqid=0 2024-11-21T01:57:50,656 INFO [M:0;b93ada835ef8:35875 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ed7da4a53b7b40c5b2f49af8abc2e2a8 2024-11-21T01:57:50,658 DEBUG [M:0;b93ada835ef8:35875 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/731456bf625a4593ae2fe21ec09e1779 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/731456bf625a4593ae2fe21ec09e1779 2024-11-21T01:57:50,659 INFO [M:0;b93ada835ef8:35875 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/731456bf625a4593ae2fe21ec09e1779, entries=8, sequenceid=84, filesize=5.5 K 2024-11-21T01:57:50,660 DEBUG [M:0;b93ada835ef8:35875 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3266b4bbb19c420bb0497da9c32513ed as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3266b4bbb19c420bb0497da9c32513ed 2024-11-21T01:57:50,661 INFO [M:0;b93ada835ef8:35875 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3266b4bbb19c420bb0497da9c32513ed, entries=9, sequenceid=84, filesize=6.5 K 2024-11-21T01:57:50,662 DEBUG [M:0;b93ada835ef8:35875 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ed7da4a53b7b40c5b2f49af8abc2e2a8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ed7da4a53b7b40c5b2f49af8abc2e2a8 2024-11-21T01:57:50,663 INFO [M:0;b93ada835ef8:35875 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ed7da4a53b7b40c5b2f49af8abc2e2a8, entries=2, sequenceid=84, filesize=5.1 K 2024-11-21T01:57:50,664 INFO [M:0;b93ada835ef8:35875 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 81ms, sequenceid=84, compaction requested=false 2024-11-21T01:57:50,666 INFO [M:0;b93ada835ef8:35875 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:57:50,666 DEBUG [M:0;b93ada835ef8:35875 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732154270583Disabling compacts and flushes for region at 1732154270583Disabling writes for close at 1732154270583Obtaining lock to block concurrent updates at 1732154270583Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732154270583Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732154270584 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732154270584Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732154270584Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732154270608 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732154270609 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732154270612 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732154270634 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732154270634Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732154270637 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732154270655 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732154270655Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3eacd68d: reopening flushed file at 1732154270658 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f172e7d: reopening flushed file at 1732154270660 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@131b1e54: reopening flushed file at 1732154270661 (+1 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 81ms, sequenceid=84, compaction requested=false at 1732154270664 (+3 ms)Writing region close event to WAL at 1732154270666 (+2 ms)Closed at 1732154270666 2024-11-21T01:57:50,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:57:50,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36587-0x10113037fa70001, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:57:50,667 INFO [RS:0;b93ada835ef8:36587 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T01:57:50,667 INFO [RS:0;b93ada835ef8:36587 {}] regionserver.HRegionServer(1031): Exiting; stopping=b93ada835ef8,36587,1732154265478; zookeeper connection closed. 2024-11-21T01:57:50,668 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T01:57:50,668 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:57:50,668 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2f91ffd4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2f91ffd4 2024-11-21T01:57:50,668 INFO [RS:1;b93ada835ef8:37077 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T01:57:50,668 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37077-0x10113037fa70002, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:57:50,668 INFO [RS:1;b93ada835ef8:37077 {}] regionserver.HRegionServer(1031): Exiting; stopping=b93ada835ef8,37077,1732154265666; zookeeper connection closed. 2024-11-21T01:57:50,668 INFO [M:0;b93ada835ef8:35875 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T01:57:50,668 INFO [M:0;b93ada835ef8:35875 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35875 2024-11-21T01:57:50,668 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@344e2191 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@344e2191 2024-11-21T01:57:50,669 INFO [M:0;b93ada835ef8:35875 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T01:57:50,669 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-21T01:57:50,771 INFO [M:0;b93ada835ef8:35875 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T01:57:50,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:57:50,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35875-0x10113037fa70000, quorum=127.0.0.1:65204, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:57:50,777 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T01:57:50,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-21T01:57:50,801 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:50,811 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: master/b93ada835ef8:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: regionserver/b93ada835ef8:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1732154270430 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/b93ada835ef8:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/b93ada835ef8:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732154270433 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1732154268740 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732154270273 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=763 (was 831), ProcessCount=11 (was 11), AvailableMemoryMB=2888 (was 3329) 2024-11-21T01:57:50,818 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=763, ProcessCount=11, AvailableMemoryMB=2887 2024-11-21T01:57:50,819 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/hadoop-log-dir so I do NOT create it in target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae 2024-11-21T01:57:50,819 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/hadoop-tmp-dir so I do NOT create it in target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae 2024-11-21T01:57:50,819 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:50,823 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/cluster_358f7c89-1d74-ac7d-ddcb-ff5f37c30029, deleteOnExit=true 2024-11-21T01:57:50,827 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/cluster_358f7c89-1d74-ac7d-ddcb-ff5f37c30029/zookeeper_0, clientPort=57707, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/cluster_358f7c89-1d74-ac7d-ddcb-ff5f37c30029/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/cluster_358f7c89-1d74-ac7d-ddcb-ff5f37c30029/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T01:57:50,829 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57707 2024-11-21T01:57:50,845 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060 with version=8 2024-11-21T01:57:50,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/hbase-staging 2024-11-21T01:57:50,848 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b93ada835ef8:0 server-side Connection retries=45 2024-11-21T01:57:50,848 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:50,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:50,849 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T01:57:50,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:50,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T01:57:50,849 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T01:57:50,849 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T01:57:50,850 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42119 2024-11-21T01:57:50,851 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42119 connecting to ZooKeeper ensemble=127.0.0.1:57707 2024-11-21T01:57:50,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:421190x0, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T01:57:50,860 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42119-0x101130399150000 connected 2024-11-21T01:57:50,875 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:50,876 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:50,877 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T01:57:50,877 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060, hbase.cluster.distributed=false 2024-11-21T01:57:50,880 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T01:57:50,881 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42119 2024-11-21T01:57:50,881 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42119 2024-11-21T01:57:50,882 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42119 2024-11-21T01:57:50,882 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42119 2024-11-21T01:57:50,882 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42119 2024-11-21T01:57:50,901 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b93ada835ef8:0 server-side Connection retries=45 2024-11-21T01:57:50,901 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:50,901 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:50,902 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T01:57:50,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:50,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T01:57:50,902 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T01:57:50,902 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T01:57:50,903 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44929 2024-11-21T01:57:50,905 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44929 connecting to ZooKeeper ensemble=127.0.0.1:57707 2024-11-21T01:57:50,905 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:50,905 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:50,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:449290x0, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T01:57:50,922 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44929-0x101130399150001 connected 2024-11-21T01:57:50,922 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T01:57:50,923 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T01:57:50,926 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/bucket.cache 2024-11-21T01:57:50,926 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T01:57:50,926 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T01:57:50,926 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T01:57:50,929 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T01:57:50,933 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T01:57:50,933 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T01:57:50,933 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T01:57:50,934 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T01:57:50,934 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/bucket.persistence is complete. 2024-11-21T01:57:50,934 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T01:57:50,936 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T01:57:50,938 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44929 2024-11-21T01:57:50,941 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44929 2024-11-21T01:57:50,944 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44929 2024-11-21T01:57:50,945 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44929 2024-11-21T01:57:50,945 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44929 2024-11-21T01:57:50,962 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b93ada835ef8:0 server-side Connection retries=45 2024-11-21T01:57:50,962 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:50,962 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:50,963 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T01:57:50,963 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:50,963 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T01:57:50,963 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T01:57:50,963 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T01:57:50,965 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40339 2024-11-21T01:57:50,966 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40339 connecting to ZooKeeper ensemble=127.0.0.1:57707 2024-11-21T01:57:50,967 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:50,967 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:50,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:403390x0, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T01:57:50,970 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:403390x0, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T01:57:50,970 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40339-0x101130399150002 connected 2024-11-21T01:57:50,970 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T01:57:50,971 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/bucket.cache 2024-11-21T01:57:50,971 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T01:57:50,972 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T01:57:50,972 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T01:57:50,972 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T01:57:50,972 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T01:57:50,973 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T01:57:50,973 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/bucket.persistence is complete. 2024-11-21T01:57:50,973 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T01:57:50,973 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T01:57:50,974 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T01:57:50,975 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T01:57:50,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40339 2024-11-21T01:57:50,979 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40339 2024-11-21T01:57:50,982 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40339 2024-11-21T01:57:50,985 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40339 2024-11-21T01:57:50,985 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40339 2024-11-21T01:57:51,005 DEBUG [M:0;b93ada835ef8:42119 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b93ada835ef8:42119 2024-11-21T01:57:51,005 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b93ada835ef8,42119,1732154270848 2024-11-21T01:57:51,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T01:57:51,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T01:57:51,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T01:57:51,010 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b93ada835ef8,42119,1732154270848 2024-11-21T01:57:51,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T01:57:51,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T01:57:51,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,012 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T01:57:51,014 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b93ada835ef8,42119,1732154270848 from backup master directory 2024-11-21T01:57:51,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T01:57:51,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T01:57:51,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b93ada835ef8,42119,1732154270848 2024-11-21T01:57:51,015 WARN [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T01:57:51,015 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b93ada835ef8,42119,1732154270848 2024-11-21T01:57:51,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T01:57:51,017 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/hbase.id] with ID: 6ec3dcf7-8e33-4c1a-a647-c87c490166cd 2024-11-21T01:57:51,017 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/.tmp/hbase.id 2024-11-21T01:57:51,024 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/hbase.id] 2024-11-21T01:57:51,063 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:51,064 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T01:57:51,065 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-21T01:57:51,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,073 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T01:57:51,074 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T01:57:51,074 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T01:57:51,098 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store 2024-11-21T01:57:51,105 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:57:51,105 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T01:57:51,105 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:57:51,105 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:57:51,105 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T01:57:51,105 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:57:51,106 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:57:51,106 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732154271105Disabling compacts and flushes for region at 1732154271105Disabling writes for close at 1732154271105Writing region close event to WAL at 1732154271105Closed at 1732154271105 2024-11-21T01:57:51,106 WARN [master/b93ada835ef8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/.initializing 2024-11-21T01:57:51,106 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/WALs/b93ada835ef8,42119,1732154270848 2024-11-21T01:57:51,112 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T01:57:51,115 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b93ada835ef8%2C42119%2C1732154270848, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/WALs/b93ada835ef8,42119,1732154270848, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/oldWALs, maxLogs=10 2024-11-21T01:57:51,133 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/WALs/b93ada835ef8,42119,1732154270848/b93ada835ef8%2C42119%2C1732154270848.1732154271116 2024-11-21T01:57:51,134 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T01:57:51,134 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:57:51,134 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:57:51,134 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:51,134 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:51,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:51,139 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T01:57:51,140 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:51,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:51,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:51,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T01:57:51,144 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:51,144 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:57:51,144 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:51,147 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T01:57:51,147 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:51,147 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:57:51,148 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:51,151 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T01:57:51,151 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:51,151 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:57:51,151 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:51,152 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:51,152 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:51,152 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:51,152 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:51,152 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T01:57:51,153 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T01:57:51,159 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T01:57:51,159 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73500966, jitterRate=0.09524974226951599}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T01:57:51,160 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732154271134Initializing all the Stores at 1732154271135 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154271135Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732154271135Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732154271135Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732154271135Cleaning up temporary data from old regions at 1732154271152 (+17 ms)Region opened successfully at 1732154271160 (+8 ms) 2024-11-21T01:57:51,160 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T01:57:51,164 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12a3f05b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b93ada835ef8/172.17.0.3:0 2024-11-21T01:57:51,165 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T01:57:51,165 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T01:57:51,165 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T01:57:51,165 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T01:57:51,166 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T01:57:51,166 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-21T01:57:51,166 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T01:57:51,168 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T01:57:51,169 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T01:57:51,170 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T01:57:51,170 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T01:57:51,171 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T01:57:51,172 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T01:57:51,173 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T01:57:51,173 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T01:57:51,175 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T01:57:51,176 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T01:57:51,177 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T01:57:51,179 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T01:57:51,180 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T01:57:51,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T01:57:51,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T01:57:51,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T01:57:51,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,183 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b93ada835ef8,42119,1732154270848, sessionid=0x101130399150000, setting cluster-up flag (Was=false) 2024-11-21T01:57:51,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,192 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T01:57:51,193 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b93ada835ef8,42119,1732154270848 2024-11-21T01:57:51,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,201 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T01:57:51,203 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b93ada835ef8,42119,1732154270848 2024-11-21T01:57:51,203 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T01:57:51,206 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T01:57:51,206 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T01:57:51,206 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T01:57:51,206 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b93ada835ef8,42119,1732154270848 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T01:57:51,208 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b93ada835ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-21T01:57:51,208 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b93ada835ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-21T01:57:51,208 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b93ada835ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-21T01:57:51,209 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b93ada835ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-21T01:57:51,209 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b93ada835ef8:0, corePoolSize=10, maxPoolSize=10 2024-11-21T01:57:51,209 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,209 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b93ada835ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-21T01:57:51,209 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,210 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T01:57:51,210 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T01:57:51,211 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:51,211 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T01:57:51,219 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732154301219 2024-11-21T01:57:51,220 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T01:57:51,220 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T01:57:51,220 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T01:57:51,220 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T01:57:51,220 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T01:57:51,220 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T01:57:51,221 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,223 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T01:57:51,223 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T01:57:51,223 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T01:57:51,226 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T01:57:51,226 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T01:57:51,226 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.large.0-1732154271226,5,FailOnTimeoutGroup] 2024-11-21T01:57:51,226 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.small.0-1732154271226,5,FailOnTimeoutGroup] 2024-11-21T01:57:51,226 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,227 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T01:57:51,227 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,227 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,234 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T01:57:51,234 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060 2024-11-21T01:57:51,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:57:51,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T01:57:51,245 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T01:57:51,245 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:51,245 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:51,245 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T01:57:51,251 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T01:57:51,251 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:51,251 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:51,251 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T01:57:51,255 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T01:57:51,255 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:51,255 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:51,255 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T01:57:51,259 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T01:57:51,259 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:51,259 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:51,259 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T01:57:51,260 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740 2024-11-21T01:57:51,260 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740 2024-11-21T01:57:51,260 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T01:57:51,260 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T01:57:51,261 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T01:57:51,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T01:57:51,278 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T01:57:51,278 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60053705, jitterRate=-0.10513006150722504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T01:57:51,279 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732154271240Initializing all the Stores at 1732154271241 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154271241Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154271241Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732154271241Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154271241Cleaning up temporary data from old regions at 1732154271260 (+19 ms)Region opened successfully at 1732154271279 (+19 ms) 2024-11-21T01:57:51,280 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T01:57:51,280 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T01:57:51,280 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T01:57:51,280 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T01:57:51,280 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T01:57:51,280 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T01:57:51,280 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732154271280Disabling compacts and flushes for region at 1732154271280Disabling writes for close at 1732154271280Writing region close event to WAL at 1732154271280Closed at 1732154271280 2024-11-21T01:57:51,282 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T01:57:51,282 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T01:57:51,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T01:57:51,283 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T01:57:51,284 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T01:57:51,291 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer(746): ClusterId : 6ec3dcf7-8e33-4c1a-a647-c87c490166cd 2024-11-21T01:57:51,291 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer(746): ClusterId : 6ec3dcf7-8e33-4c1a-a647-c87c490166cd 2024-11-21T01:57:51,291 DEBUG [RS:1;b93ada835ef8:40339 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T01:57:51,291 DEBUG [RS:0;b93ada835ef8:44929 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T01:57:51,294 DEBUG [RS:1;b93ada835ef8:40339 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T01:57:51,294 DEBUG [RS:1;b93ada835ef8:40339 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T01:57:51,295 DEBUG [RS:0;b93ada835ef8:44929 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T01:57:51,295 DEBUG [RS:0;b93ada835ef8:44929 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T01:57:51,297 DEBUG [RS:0;b93ada835ef8:44929 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T01:57:51,298 DEBUG [RS:1;b93ada835ef8:40339 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T01:57:51,298 DEBUG [RS:1;b93ada835ef8:40339 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e8be80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b93ada835ef8/172.17.0.3:0 2024-11-21T01:57:51,298 DEBUG [RS:0;b93ada835ef8:44929 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1544a41d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b93ada835ef8/172.17.0.3:0 2024-11-21T01:57:51,314 DEBUG [RS:1;b93ada835ef8:40339 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;b93ada835ef8:40339 2024-11-21T01:57:51,314 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T01:57:51,314 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T01:57:51,314 DEBUG [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T01:57:51,317 DEBUG [RS:0;b93ada835ef8:44929 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b93ada835ef8:44929 2024-11-21T01:57:51,317 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T01:57:51,317 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T01:57:51,317 DEBUG [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T01:57:51,318 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer(2659): reportForDuty to master=b93ada835ef8,42119,1732154270848 with port=40339, startcode=1732154270962 2024-11-21T01:57:51,318 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer(2659): reportForDuty to master=b93ada835ef8,42119,1732154270848 with port=44929, startcode=1732154270901 2024-11-21T01:57:51,318 DEBUG [RS:1;b93ada835ef8:40339 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T01:57:51,318 DEBUG [RS:0;b93ada835ef8:44929 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T01:57:51,323 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35607, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T01:57:51,324 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56569, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T01:57:51,324 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42119 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b93ada835ef8,44929,1732154270901 2024-11-21T01:57:51,324 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42119 {}] master.ServerManager(517): Registering regionserver=b93ada835ef8,44929,1732154270901 2024-11-21T01:57:51,326 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42119 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b93ada835ef8,40339,1732154270962 2024-11-21T01:57:51,326 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42119 {}] master.ServerManager(517): Registering regionserver=b93ada835ef8,40339,1732154270962 2024-11-21T01:57:51,326 DEBUG [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060 2024-11-21T01:57:51,326 DEBUG [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T01:57:51,326 DEBUG [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T01:57:51,327 DEBUG [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060 2024-11-21T01:57:51,327 DEBUG [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T01:57:51,327 DEBUG [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T01:57:51,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T01:57:51,332 DEBUG [RS:0;b93ada835ef8:44929 {}] zookeeper.ZKUtil(111): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b93ada835ef8,44929,1732154270901 2024-11-21T01:57:51,332 WARN [RS:0;b93ada835ef8:44929 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T01:57:51,332 INFO [RS:0;b93ada835ef8:44929 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T01:57:51,332 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b93ada835ef8,40339,1732154270962] 2024-11-21T01:57:51,332 DEBUG [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,44929,1732154270901 2024-11-21T01:57:51,332 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b93ada835ef8,44929,1732154270901] 2024-11-21T01:57:51,332 DEBUG [RS:1;b93ada835ef8:40339 {}] zookeeper.ZKUtil(111): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b93ada835ef8,40339,1732154270962 2024-11-21T01:57:51,333 WARN [RS:1;b93ada835ef8:40339 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T01:57:51,333 INFO [RS:1;b93ada835ef8:40339 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T01:57:51,333 DEBUG [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,40339,1732154270962 2024-11-21T01:57:51,339 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T01:57:51,339 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T01:57:51,341 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T01:57:51,343 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T01:57:51,343 INFO [RS:1;b93ada835ef8:40339 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T01:57:51,343 INFO [RS:1;b93ada835ef8:40339 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,343 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T01:57:51,343 INFO [RS:0;b93ada835ef8:44929 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T01:57:51,343 INFO [RS:0;b93ada835ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,344 INFO [RS:1;b93ada835ef8:40339 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T01:57:51,344 INFO [RS:1;b93ada835ef8:40339 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,344 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T01:57:51,344 DEBUG [RS:1;b93ada835ef8:40339 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,344 DEBUG [RS:1;b93ada835ef8:40339 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,344 DEBUG [RS:1;b93ada835ef8:40339 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,344 DEBUG [RS:1;b93ada835ef8:40339 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,344 INFO [RS:0;b93ada835ef8:44929 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T01:57:51,344 DEBUG [RS:1;b93ada835ef8:40339 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,345 INFO [RS:0;b93ada835ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,345 DEBUG [RS:1;b93ada835ef8:40339 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b93ada835ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-21T01:57:51,345 DEBUG [RS:1;b93ada835ef8:40339 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,345 DEBUG [RS:0;b93ada835ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,345 DEBUG [RS:1;b93ada835ef8:40339 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,345 DEBUG [RS:0;b93ada835ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,345 DEBUG [RS:1;b93ada835ef8:40339 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,345 DEBUG [RS:0;b93ada835ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,345 DEBUG [RS:1;b93ada835ef8:40339 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,345 DEBUG [RS:0;b93ada835ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,345 DEBUG [RS:1;b93ada835ef8:40339 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,345 DEBUG [RS:0;b93ada835ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,345 DEBUG [RS:0;b93ada835ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b93ada835ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-21T01:57:51,345 DEBUG [RS:1;b93ada835ef8:40339 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,345 DEBUG [RS:0;b93ada835ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,345 DEBUG [RS:1;b93ada835ef8:40339 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b93ada835ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-21T01:57:51,346 DEBUG [RS:1;b93ada835ef8:40339 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-21T01:57:51,346 DEBUG [RS:0;b93ada835ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,346 DEBUG [RS:0;b93ada835ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,346 DEBUG [RS:0;b93ada835ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,346 DEBUG [RS:0;b93ada835ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,346 DEBUG [RS:0;b93ada835ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:51,346 DEBUG [RS:0;b93ada835ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b93ada835ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-21T01:57:51,347 DEBUG [RS:0;b93ada835ef8:44929 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-21T01:57:51,347 INFO [RS:1;b93ada835ef8:40339 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,347 INFO [RS:1;b93ada835ef8:40339 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,347 INFO [RS:1;b93ada835ef8:40339 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,347 INFO [RS:1;b93ada835ef8:40339 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,347 INFO [RS:1;b93ada835ef8:40339 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,347 INFO [RS:1;b93ada835ef8:40339 {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,40339,1732154270962-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T01:57:51,347 INFO [RS:0;b93ada835ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,347 INFO [RS:0;b93ada835ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,347 INFO [RS:0;b93ada835ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,347 INFO [RS:0;b93ada835ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,347 INFO [RS:0;b93ada835ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,348 INFO [RS:0;b93ada835ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,44929,1732154270901-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T01:57:51,370 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T01:57:51,370 INFO [RS:1;b93ada835ef8:40339 {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,40339,1732154270962-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,370 INFO [RS:1;b93ada835ef8:40339 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,370 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.Replication(171): b93ada835ef8,40339,1732154270962 started 2024-11-21T01:57:51,370 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T01:57:51,371 INFO [RS:0;b93ada835ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,44929,1732154270901-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,371 INFO [RS:0;b93ada835ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,371 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.Replication(171): b93ada835ef8,44929,1732154270901 started 2024-11-21T01:57:51,386 INFO [RS:1;b93ada835ef8:40339 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,386 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer(1482): Serving as b93ada835ef8,40339,1732154270962, RpcServer on b93ada835ef8/172.17.0.3:40339, sessionid=0x101130399150002 2024-11-21T01:57:51,386 INFO [RS:0;b93ada835ef8:44929 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,386 DEBUG [RS:1;b93ada835ef8:40339 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T01:57:51,386 DEBUG [RS:1;b93ada835ef8:40339 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b93ada835ef8,40339,1732154270962 2024-11-21T01:57:51,386 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer(1482): Serving as b93ada835ef8,44929,1732154270901, RpcServer on b93ada835ef8/172.17.0.3:44929, sessionid=0x101130399150001 2024-11-21T01:57:51,386 DEBUG [RS:1;b93ada835ef8:40339 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b93ada835ef8,40339,1732154270962' 2024-11-21T01:57:51,386 DEBUG [RS:1;b93ada835ef8:40339 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T01:57:51,386 DEBUG [RS:0;b93ada835ef8:44929 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T01:57:51,386 DEBUG [RS:0;b93ada835ef8:44929 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b93ada835ef8,44929,1732154270901 2024-11-21T01:57:51,386 DEBUG [RS:0;b93ada835ef8:44929 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b93ada835ef8,44929,1732154270901' 2024-11-21T01:57:51,386 DEBUG [RS:0;b93ada835ef8:44929 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T01:57:51,387 DEBUG [RS:1;b93ada835ef8:40339 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T01:57:51,387 DEBUG [RS:0;b93ada835ef8:44929 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T01:57:51,387 DEBUG [RS:0;b93ada835ef8:44929 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T01:57:51,387 DEBUG [RS:1;b93ada835ef8:40339 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T01:57:51,387 DEBUG [RS:0;b93ada835ef8:44929 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T01:57:51,387 DEBUG [RS:1;b93ada835ef8:40339 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T01:57:51,387 DEBUG [RS:0;b93ada835ef8:44929 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b93ada835ef8,44929,1732154270901 2024-11-21T01:57:51,387 DEBUG [RS:1;b93ada835ef8:40339 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b93ada835ef8,40339,1732154270962 2024-11-21T01:57:51,387 DEBUG [RS:0;b93ada835ef8:44929 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b93ada835ef8,44929,1732154270901' 2024-11-21T01:57:51,388 DEBUG [RS:1;b93ada835ef8:40339 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b93ada835ef8,40339,1732154270962' 2024-11-21T01:57:51,388 DEBUG [RS:0;b93ada835ef8:44929 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T01:57:51,388 DEBUG [RS:1;b93ada835ef8:40339 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T01:57:51,388 DEBUG [RS:0;b93ada835ef8:44929 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T01:57:51,388 DEBUG [RS:1;b93ada835ef8:40339 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T01:57:51,389 DEBUG [RS:0;b93ada835ef8:44929 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T01:57:51,389 INFO [RS:0;b93ada835ef8:44929 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T01:57:51,389 INFO [RS:0;b93ada835ef8:44929 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T01:57:51,389 DEBUG [RS:1;b93ada835ef8:40339 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T01:57:51,389 INFO [RS:1;b93ada835ef8:40339 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T01:57:51,389 INFO [RS:1;b93ada835ef8:40339 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T01:57:51,435 WARN [b93ada835ef8:42119 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T01:57:51,489 INFO [RS:0;b93ada835ef8:44929 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T01:57:51,490 INFO [RS:1;b93ada835ef8:40339 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T01:57:51,490 INFO [RS:0;b93ada835ef8:44929 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=b93ada835ef8%2C44929%2C1732154270901, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,44929,1732154270901, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/oldWALs, maxLogs=55 2024-11-21T01:57:51,490 INFO [RS:1;b93ada835ef8:40339 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=b93ada835ef8%2C40339%2C1732154270962, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,40339,1732154270962, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/oldWALs, maxLogs=55 2024-11-21T01:57:51,514 INFO [RS:1;b93ada835ef8:40339 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,40339,1732154270962/b93ada835ef8%2C40339%2C1732154270962.1732154271493 2024-11-21T01:57:51,514 INFO [RS:0;b93ada835ef8:44929 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,44929,1732154270901/b93ada835ef8%2C44929%2C1732154270901.1732154271493 2024-11-21T01:57:51,514 DEBUG [RS:1;b93ada835ef8:40339 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T01:57:51,515 DEBUG [RS:0;b93ada835ef8:44929 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T01:57:51,685 DEBUG [b93ada835ef8:42119 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-21T01:57:51,685 DEBUG [b93ada835ef8:42119 {}] balancer.BalancerClusterState(204): Hosts are {b93ada835ef8=0} racks are {/default-rack=0} 2024-11-21T01:57:51,686 DEBUG [b93ada835ef8:42119 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T01:57:51,686 DEBUG [b93ada835ef8:42119 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T01:57:51,686 DEBUG [b93ada835ef8:42119 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T01:57:51,686 DEBUG [b93ada835ef8:42119 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T01:57:51,686 INFO [b93ada835ef8:42119 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T01:57:51,686 INFO [b93ada835ef8:42119 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T01:57:51,686 DEBUG [b93ada835ef8:42119 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T01:57:51,687 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b93ada835ef8,44929,1732154270901 2024-11-21T01:57:51,688 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b93ada835ef8,44929,1732154270901, state=OPENING 2024-11-21T01:57:51,689 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T01:57:51,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:51,691 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T01:57:51,691 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:51,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b93ada835ef8,44929,1732154270901}] 2024-11-21T01:57:51,692 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:51,692 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:51,844 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T01:57:51,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51801, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T01:57:51,849 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T01:57:51,849 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T01:57:51,850 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T01:57:51,851 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=b93ada835ef8%2C44929%2C1732154270901.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,44929,1732154270901, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/oldWALs, maxLogs=55 2024-11-21T01:57:51,881 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,44929,1732154270901/b93ada835ef8%2C44929%2C1732154270901.meta.1732154271852.meta 2024-11-21T01:57:51,882 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T01:57:51,882 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:57:51,882 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T01:57:51,882 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T01:57:51,883 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T01:57:51,883 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T01:57:51,883 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:57:51,883 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T01:57:51,883 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T01:57:51,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T01:57:51,885 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T01:57:51,885 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:51,886 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:51,886 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T01:57:51,886 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T01:57:51,886 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:51,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:51,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T01:57:51,887 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T01:57:51,887 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:51,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:51,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T01:57:51,888 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T01:57:51,888 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:51,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:51,889 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T01:57:51,890 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740 2024-11-21T01:57:51,890 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740 2024-11-21T01:57:51,891 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T01:57:51,891 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T01:57:51,891 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T01:57:51,892 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T01:57:51,893 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65625621, jitterRate=-0.02210204303264618}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T01:57:51,893 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T01:57:51,894 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732154271883Writing region info on filesystem at 1732154271883Initializing all the Stores at 1732154271884 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154271884Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154271884Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732154271884Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154271884Cleaning up temporary data from old regions at 1732154271891 (+7 ms)Running coprocessor post-open hooks at 1732154271893 (+2 ms)Region opened successfully at 1732154271894 (+1 ms) 2024-11-21T01:57:51,894 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732154271844 2024-11-21T01:57:51,898 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b93ada835ef8,44929,1732154270901 2024-11-21T01:57:51,898 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T01:57:51,898 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T01:57:51,899 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b93ada835ef8,44929,1732154270901, state=OPEN 2024-11-21T01:57:51,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T01:57:51,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T01:57:51,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T01:57:51,901 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b93ada835ef8,44929,1732154270901 2024-11-21T01:57:51,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:51,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:51,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:51,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T01:57:51,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b93ada835ef8,44929,1732154270901 in 210 msec 2024-11-21T01:57:51,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T01:57:51,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 622 msec 2024-11-21T01:57:51,911 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T01:57:51,911 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T01:57:51,916 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:57:51,916 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,44929,1732154270901, seqNum=-1] 2024-11-21T01:57:51,916 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:57:51,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57047, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:57:51,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 719 msec 2024-11-21T01:57:51,925 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732154271925, completionTime=-1 2024-11-21T01:57:51,925 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-21T01:57:51,925 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T01:57:51,928 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T01:57:51,928 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732154331928 2024-11-21T01:57:51,928 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732154391928 2024-11-21T01:57:51,928 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-21T01:57:51,928 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-21T01:57:51,929 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,42119,1732154270848-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,929 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,42119,1732154270848-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,929 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,42119,1732154270848-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,929 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b93ada835ef8:42119, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,929 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,929 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,932 DEBUG [master/b93ada835ef8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T01:57:51,934 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.919sec 2024-11-21T01:57:51,934 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T01:57:51,934 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T01:57:51,934 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T01:57:51,934 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T01:57:51,934 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T01:57:51,934 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,42119,1732154270848-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T01:57:51,934 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,42119,1732154270848-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T01:57:51,935 DEBUG [master/b93ada835ef8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T01:57:51,935 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T01:57:51,935 INFO [master/b93ada835ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,42119,1732154270848-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:51,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e1a2ccd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:57:51,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b93ada835ef8,42119,-1 for getting cluster id 2024-11-21T01:57:51,991 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T01:57:51,992 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6ec3dcf7-8e33-4c1a-a647-c87c490166cd' 2024-11-21T01:57:51,993 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T01:57:51,993 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6ec3dcf7-8e33-4c1a-a647-c87c490166cd" 2024-11-21T01:57:51,993 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6729636, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:57:51,993 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b93ada835ef8,42119,-1] 2024-11-21T01:57:51,994 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T01:57:51,994 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:51,995 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T01:57:51,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d25b74f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T01:57:51,997 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:57:51,999 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,44929,1732154270901, seqNum=-1] 2024-11-21T01:57:51,999 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:57:52,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:57:52,004 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b93ada835ef8,42119,1732154270848 2024-11-21T01:57:52,004 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T01:57:52,006 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b93ada835ef8,42119,1732154270848 2024-11-21T01:57:52,006 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4c12e8a8 2024-11-21T01:57:52,006 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T01:57:52,008 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T01:57:52,009 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42119 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testBlockEvictionOnGracefulStop1732154272004', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T01:57:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42119 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732154272004 2024-11-21T01:57:52,011 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732154272004 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T01:57:52,011 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:52,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42119 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732154272004" procId is: 4 2024-11-21T01:57:52,012 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732154272004 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T01:57:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42119 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T01:57:52,023 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732154272004-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7c660ebcf5932b16019bde5cc2d9649d, NAME => 'testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732154272004', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060 2024-11-21T01:57:52,030 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732154272004-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:57:52,030 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732154272004-pool-0 {}] regionserver.HRegion(1722): Closing 7c660ebcf5932b16019bde5cc2d9649d, disabling compactions & flushes 2024-11-21T01:57:52,030 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732154272004-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:52,030 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732154272004-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:52,030 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732154272004-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. after waiting 0 ms 2024-11-21T01:57:52,030 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732154272004-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:52,030 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732154272004-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:52,030 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732154272004-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7c660ebcf5932b16019bde5cc2d9649d: Waiting for close lock at 1732154272030Disabling compacts and flushes for region at 1732154272030Disabling writes for close at 1732154272030Writing region close event to WAL at 1732154272030Closed at 1732154272030 2024-11-21T01:57:52,032 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732154272004 execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T01:57:52,032 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732154272032"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732154272032"}]},"ts":"1732154272032"} 2024-11-21T01:57:52,034 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T01:57:52,035 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732154272004 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T01:57:52,036 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732154272004","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732154272035"}]},"ts":"1732154272035"} 2024-11-21T01:57:52,038 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732154272004, state=ENABLING in hbase:meta 2024-11-21T01:57:52,038 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {b93ada835ef8=0} racks are {/default-rack=0} 2024-11-21T01:57:52,038 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T01:57:52,039 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T01:57:52,039 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T01:57:52,039 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T01:57:52,039 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T01:57:52,039 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T01:57:52,039 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T01:57:52,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732154272004, region=7c660ebcf5932b16019bde5cc2d9649d, ASSIGN}] 2024-11-21T01:57:52,040 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732154272004, region=7c660ebcf5932b16019bde5cc2d9649d, ASSIGN 2024-11-21T01:57:52,041 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732154272004, region=7c660ebcf5932b16019bde5cc2d9649d, ASSIGN; state=OFFLINE, location=b93ada835ef8,44929,1732154270901; forceNewPlan=false, retain=false 2024-11-21T01:57:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42119 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T01:57:52,192 INFO [b93ada835ef8:42119 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T01:57:52,192 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7c660ebcf5932b16019bde5cc2d9649d, regionState=OPENING, regionLocation=b93ada835ef8,44929,1732154270901 2024-11-21T01:57:52,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732154272004, region=7c660ebcf5932b16019bde5cc2d9649d, ASSIGN because future has completed 2024-11-21T01:57:52,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c660ebcf5932b16019bde5cc2d9649d, server=b93ada835ef8,44929,1732154270901}] 2024-11-21T01:57:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42119 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T01:57:52,351 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:52,351 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c660ebcf5932b16019bde5cc2d9649d, NAME => 'testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:57:52,351 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732154272004 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:52,352 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:57:52,352 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:52,352 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:52,353 INFO [StoreOpener-7c660ebcf5932b16019bde5cc2d9649d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:52,357 INFO [StoreOpener-7c660ebcf5932b16019bde5cc2d9649d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c660ebcf5932b16019bde5cc2d9649d columnFamilyName family 2024-11-21T01:57:52,357 DEBUG [StoreOpener-7c660ebcf5932b16019bde5cc2d9649d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:52,358 INFO [StoreOpener-7c660ebcf5932b16019bde5cc2d9649d-1 {}] regionserver.HStore(327): Store=7c660ebcf5932b16019bde5cc2d9649d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:57:52,358 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:52,359 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:52,359 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:52,359 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:52,359 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:52,360 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:52,366 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T01:57:52,367 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7c660ebcf5932b16019bde5cc2d9649d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73860934, jitterRate=0.10061368346214294}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T01:57:52,367 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:52,368 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7c660ebcf5932b16019bde5cc2d9649d: Running coprocessor pre-open hook at 1732154272352Writing region info on filesystem at 1732154272352Initializing all the Stores at 1732154272352Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732154272352Cleaning up temporary data from old regions at 1732154272359 (+7 ms)Running coprocessor post-open hooks at 1732154272367 (+8 ms)Region opened successfully at 1732154272368 (+1 ms) 2024-11-21T01:57:52,368 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d., pid=6, masterSystemTime=1732154272348 2024-11-21T01:57:52,371 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:52,371 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:52,372 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7c660ebcf5932b16019bde5cc2d9649d, regionState=OPEN, openSeqNum=2, regionLocation=b93ada835ef8,44929,1732154270901 2024-11-21T01:57:52,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c660ebcf5932b16019bde5cc2d9649d, server=b93ada835ef8,44929,1732154270901 because future has completed 2024-11-21T01:57:52,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T01:57:52,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c660ebcf5932b16019bde5cc2d9649d, server=b93ada835ef8,44929,1732154270901 in 180 msec 2024-11-21T01:57:52,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T01:57:52,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732154272004, region=7c660ebcf5932b16019bde5cc2d9649d, ASSIGN in 339 msec 2024-11-21T01:57:52,381 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732154272004 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T01:57:52,381 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732154272004","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732154272381"}]},"ts":"1732154272381"} 2024-11-21T01:57:52,383 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732154272004, state=ENABLED in hbase:meta 2024-11-21T01:57:52,384 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732154272004 execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T01:57:52,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732154272004 in 375 msec 2024-11-21T01:57:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42119 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T01:57:52,640 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732154272004 get assigned. Timeout = 60000ms 2024-11-21T01:57:52,640 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732154272004 completed 2024-11-21T01:57:52,641 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T01:57:52,645 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732154272004 assigned to meta. Checking AM states. 2024-11-21T01:57:52,645 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T01:57:52,645 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732154272004 assigned. 2024-11-21T01:57:52,649 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732154272004', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d., hostname=b93ada835ef8,44929,1732154270901, seqNum=2] 2024-11-21T01:57:52,654 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7c660ebcf5932b16019bde5cc2d9649d 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-21T01:57:52,666 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5fe9c79b4892475c8ca4222f4a1aa903 2024-11-21T01:57:52,679 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/.tmp/family/5fe9c79b4892475c8ca4222f4a1aa903 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-21T01:57:52,679 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/.tmp/family/5fe9c79b4892475c8ca4222f4a1aa903 2024-11-21T01:57:52,682 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/.tmp/family/5fe9c79b4892475c8ca4222f4a1aa903 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/family/5fe9c79b4892475c8ca4222f4a1aa903 2024-11-21T01:57:52,683 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/family/5fe9c79b4892475c8ca4222f4a1aa903, delay=982 ms 2024-11-21T01:57:52,684 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/family/5fe9c79b4892475c8ca4222f4a1aa903, entries=2, sequenceid=6, filesize=4.9 K 2024-11-21T01:57:52,685 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 7c660ebcf5932b16019bde5cc2d9649d in 31ms, sequenceid=6, compaction requested=false 2024-11-21T01:57:52,685 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732154272004' 2024-11-21T01:57:52,686 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7c660ebcf5932b16019bde5cc2d9649d: 2024-11-21T01:57:53,054 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T01:57:53,666 DEBUG [hfile-prefetch-1732154270273 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 5fe9c79b4892475c8ca4222f4a1aa903_0. 2024-11-21T01:57:53,667 DEBUG [hfile-prefetch-1732154270273 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 5fe9c79b4892475c8ca4222f4a1aa903_117. 2024-11-21T01:57:53,667 DEBUG [hfile-prefetch-1732154270273 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/family/5fe9c79b4892475c8ca4222f4a1aa903, with total blocks 2, and data blocks 1 2024-11-21T01:57:53,667 DEBUG [hfile-prefetch-1732154270273 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-21T01:57:53,667 DEBUG [hfile-prefetch-1732154270273 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/family/5fe9c79b4892475c8ca4222f4a1aa903, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-21T01:57:53,667 DEBUG [hfile-prefetch-1732154270273 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/family/5fe9c79b4892475c8ca4222f4a1aa903. Will sleep 100ms and try the verification again. 2024-11-21T01:57:53,678 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T01:57:53,678 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T01:57:53,678 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T01:57:53,678 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T01:57:53,681 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T01:57:53,681 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732154273678 2024-11-21T01:57:53,682 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T01:57:53,686 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;b93ada835ef8:44929,5,FailOnTimeoutGroup] 2024-11-21T01:57:53,687 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b93ada835ef8,44929,1732154270901' ***** 2024-11-21T01:57:53,687 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-21T01:57:53,687 INFO [RS:0;b93ada835ef8:44929 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T01:57:53,687 INFO [RS:0;b93ada835ef8:44929 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/bucket.persistence 2024-11-21T01:57:53,687 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T01:57:53,687 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T01:57:53,687 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T01:57:53,687 INFO [RS:0;b93ada835ef8:44929 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T01:57:53,687 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T01:57:53,697 DEBUG [RS:0;b93ada835ef8:44929 {}] bucket.BucketCache(1487): Thread RS:0;b93ada835ef8:44929 started persisting bucket cache to file 2024-11-21T01:57:53,698 DEBUG [RS:0;b93ada835ef8:44929 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T01:57:53,698 DEBUG [RS:0;b93ada835ef8:44929 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T01:57:53,701 DEBUG [RS:0;b93ada835ef8:44929 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-21T01:57:53,701 DEBUG [RS:0;b93ada835ef8:44929 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732154273697 2024-11-21T01:57:53,701 DEBUG [RS:0;b93ada835ef8:44929 {}] bucket.BucketCache(1509): Thread RS:0;b93ada835ef8:44929 finished persisting bucket cache to file, renaming 2024-11-21T01:57:53,701 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T01:57:53,701 INFO [RS:0;b93ada835ef8:44929 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T01:57:53,701 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T01:57:53,701 INFO [RS:0;b93ada835ef8:44929 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T01:57:53,701 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer(3091): Received CLOSE for 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:53,702 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer(959): stopping server b93ada835ef8,44929,1732154270901 2024-11-21T01:57:53,702 INFO [RS:0;b93ada835ef8:44929 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T01:57:53,702 INFO [RS:0;b93ada835ef8:44929 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b93ada835ef8:44929. 2024-11-21T01:57:53,702 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T01:57:53,702 DEBUG [RS:0;b93ada835ef8:44929 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:57:53,702 DEBUG [RS:0;b93ada835ef8:44929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:53,702 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7c660ebcf5932b16019bde5cc2d9649d, disabling compactions & flushes 2024-11-21T01:57:53,702 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:53,702 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T01:57:53,702 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:53,702 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T01:57:53,702 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. after waiting 0 ms 2024-11-21T01:57:53,702 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T01:57:53,702 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:53,702 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T01:57:53,703 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-21T01:57:53,703 DEBUG [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 7c660ebcf5932b16019bde5cc2d9649d=testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d.} 2024-11-21T01:57:53,703 DEBUG [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:53,703 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T01:57:53,703 DEBUG [StoreFileCloser-7c660ebcf5932b16019bde5cc2d9649d-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/family/5fe9c79b4892475c8ca4222f4a1aa903 2024-11-21T01:57:53,703 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T01:57:53,703 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T01:57:53,703 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T01:57:53,703 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T01:57:53,703 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T01:57:53,703 WARN [hfile-prefetch-1732154270273 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/family/5fe9c79b4892475c8ca4222f4a1aa903, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-21T01:57:53,703 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-21T01:57:53,703 DEBUG [hfile-prefetch-1732154270273 {}] hfile.PrefetchExecutor(121): Prefetch completed for 5fe9c79b4892475c8ca4222f4a1aa903 2024-11-21T01:57:53,708 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-21T01:57:53,708 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:53,708 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7c660ebcf5932b16019bde5cc2d9649d: Waiting for close lock at 1732154273702Running coprocessor pre-close hooks at 1732154273702Disabling compacts and flushes for region at 1732154273702Disabling writes for close at 1732154273702Writing region close event to WAL at 1732154273703 (+1 ms)Running coprocessor post-close hooks at 1732154273708 (+5 ms)Closed at 1732154273708 2024-11-21T01:57:53,708 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:53,714 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e16e834639254555ae1e0445bb27ee72 2024-11-21T01:57:53,729 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/info/e16e834639254555ae1e0445bb27ee72 is 209, key is testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d./info:regioninfo/1732154272372/Put/seqid=0 2024-11-21T01:57:53,730 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/info/e16e834639254555ae1e0445bb27ee72 2024-11-21T01:57:53,740 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 54d3cc8db22543569f9c9d4f43926cea 2024-11-21T01:57:53,750 INFO [regionserver/b93ada835ef8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T01:57:53,756 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/ns/54d3cc8db22543569f9c9d4f43926cea is 43, key is default/ns:d/1732154271920/Put/seqid=0 2024-11-21T01:57:53,757 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/ns/54d3cc8db22543569f9c9d4f43926cea 2024-11-21T01:57:53,767 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 98a02fe0e9cc420aa364db87e5562208 2024-11-21T01:57:53,781 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/table/98a02fe0e9cc420aa364db87e5562208 is 80, key is testBlockEvictionOnGracefulStop1732154272004/table:state/1732154272381/Put/seqid=0 2024-11-21T01:57:53,781 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/table/98a02fe0e9cc420aa364db87e5562208 2024-11-21T01:57:53,784 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/info/e16e834639254555ae1e0445bb27ee72 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/info/e16e834639254555ae1e0445bb27ee72 2024-11-21T01:57:53,785 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/info/e16e834639254555ae1e0445bb27ee72, delay=945 ms 2024-11-21T01:57:53,786 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/info/e16e834639254555ae1e0445bb27ee72, entries=10, sequenceid=11, filesize=7.0 K 2024-11-21T01:57:53,787 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/ns/54d3cc8db22543569f9c9d4f43926cea as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/ns/54d3cc8db22543569f9c9d4f43926cea 2024-11-21T01:57:53,788 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/ns/54d3cc8db22543569f9c9d4f43926cea, delay=965 ms 2024-11-21T01:57:53,789 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/ns/54d3cc8db22543569f9c9d4f43926cea, entries=2, sequenceid=11, filesize=5.0 K 2024-11-21T01:57:53,789 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/table/98a02fe0e9cc420aa364db87e5562208 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/table/98a02fe0e9cc420aa364db87e5562208 2024-11-21T01:57:53,790 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/table/98a02fe0e9cc420aa364db87e5562208, delay=972 ms 2024-11-21T01:57:53,791 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/table/98a02fe0e9cc420aa364db87e5562208, entries=2, sequenceid=11, filesize=5.3 K 2024-11-21T01:57:53,792 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 89ms, sequenceid=11, compaction requested=false 2024-11-21T01:57:53,794 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/info/e16e834639254555ae1e0445bb27ee72 2024-11-21T01:57:53,794 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/ns/54d3cc8db22543569f9c9d4f43926cea 2024-11-21T01:57:53,795 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/table/98a02fe0e9cc420aa364db87e5562208 2024-11-21T01:57:53,800 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-21T01:57:53,800 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T01:57:53,800 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T01:57:53,801 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732154273703Running coprocessor pre-close hooks at 1732154273703Disabling compacts and flushes for region at 1732154273703Disabling writes for close at 1732154273703Obtaining lock to block concurrent updates at 1732154273703Preparing flush snapshotting stores in 1588230740 at 1732154273703Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732154273704 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732154273704Flushing 1588230740/info: creating writer at 1732154273704Flushing 1588230740/info: appending metadata at 1732154273729 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732154273729Flushing 1588230740/ns: creating writer at 1732154273733 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732154273755 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1732154273755Flushing 1588230740/table: creating writer at 1732154273760 (+5 ms)Flushing 1588230740/table: appending metadata at 1732154273780 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732154273780Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ac2efea: reopening flushed file at 1732154273784 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d965ad0: reopening flushed file at 1732154273787 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a65685d: reopening flushed file at 1732154273789 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 89ms, sequenceid=11, compaction requested=false at 1732154273792 (+3 ms)Writing region close event to WAL at 1732154273795 (+3 ms)Running coprocessor post-close hooks at 1732154273800 (+5 ms)Closed at 1732154273800 2024-11-21T01:57:53,801 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T01:57:53,903 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer(976): stopping server b93ada835ef8,44929,1732154270901; all regions closed. 2024-11-21T01:57:53,905 DEBUG [RS:0;b93ada835ef8:44929 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/oldWALs 2024-11-21T01:57:53,905 INFO [RS:0;b93ada835ef8:44929 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b93ada835ef8%2C44929%2C1732154270901.meta:.meta(num 1732154271852) 2024-11-21T01:57:53,907 DEBUG [RS:0;b93ada835ef8:44929 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/oldWALs 2024-11-21T01:57:53,907 INFO [RS:0;b93ada835ef8:44929 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b93ada835ef8%2C44929%2C1732154270901:(num 1732154271493) 2024-11-21T01:57:53,907 DEBUG [RS:0;b93ada835ef8:44929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:53,907 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T01:57:53,907 INFO [RS:0;b93ada835ef8:44929 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T01:57:53,908 INFO [RS:0;b93ada835ef8:44929 {}] hbase.ChoreService(370): Chore service for: regionserver/b93ada835ef8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T01:57:53,908 INFO [RS:0;b93ada835ef8:44929 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T01:57:53,908 INFO [regionserver/b93ada835ef8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T01:57:53,908 INFO [RS:0;b93ada835ef8:44929 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44929 2024-11-21T01:57:53,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T01:57:53,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b93ada835ef8,44929,1732154270901 2024-11-21T01:57:53,912 INFO [RS:0;b93ada835ef8:44929 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T01:57:53,914 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b93ada835ef8,44929,1732154270901] 2024-11-21T01:57:53,915 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b93ada835ef8,44929,1732154270901 already deleted, retry=false 2024-11-21T01:57:53,916 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of b93ada835ef8,44929,1732154270901 on b93ada835ef8,42119,1732154270848 2024-11-21T01:57:53,920 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure b93ada835ef8,44929,1732154270901, splitWal=true, meta=true 2024-11-21T01:57:53,922 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure b93ada835ef8,44929,1732154270901, splitWal=true, meta=true 2024-11-21T01:57:53,922 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for b93ada835ef8,44929,1732154270901 (carryingMeta=true) b93ada835ef8,44929,1732154270901/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3c86cbc2[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-21T01:57:53,923 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure b93ada835ef8,44929,1732154270901, splitWal=true, meta=true, isMeta: true 2024-11-21T01:57:53,924 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,44929,1732154270901-splitting 2024-11-21T01:57:53,924 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,44929,1732154270901-splitting dir is empty, no logs to split. 2024-11-21T01:57:53,924 INFO [PEWorker-2 {}] master.SplitWALManager(105): b93ada835ef8,44929,1732154270901 WAL count=0, meta=true 2024-11-21T01:57:53,926 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,44929,1732154270901-splitting dir is empty, no logs to split. 2024-11-21T01:57:53,926 INFO [PEWorker-2 {}] master.SplitWALManager(105): b93ada835ef8,44929,1732154270901 WAL count=0, meta=true 2024-11-21T01:57:53,926 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if b93ada835ef8,44929,1732154270901 WAL splitting is done? wals=0, meta=true 2024-11-21T01:57:53,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T01:57:53,928 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T01:57:53,929 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T01:57:54,014 INFO [RS:0;b93ada835ef8:44929 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T01:57:54,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:57:54,014 INFO [RS:0;b93ada835ef8:44929 {}] regionserver.HRegionServer(1031): Exiting; stopping=b93ada835ef8,44929,1732154270901; zookeeper connection closed. 2024-11-21T01:57:54,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44929-0x101130399150001, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:57:54,015 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2cd22344 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2cd22344 2024-11-21T01:57:54,080 DEBUG [b93ada835ef8:42119 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-21T01:57:54,080 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b93ada835ef8,40339,1732154270962 2024-11-21T01:57:54,081 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b93ada835ef8,40339,1732154270962, state=OPENING 2024-11-21T01:57:54,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T01:57:54,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T01:57:54,083 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T01:57:54,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:54,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:54,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b93ada835ef8,40339,1732154270962}] 2024-11-21T01:57:54,213 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b93ada835ef8:0 server-side Connection retries=45 2024-11-21T01:57:54,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:54,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:54,213 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T01:57:54,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T01:57:54,213 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T01:57:54,213 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T01:57:54,214 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T01:57:54,214 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35269 2024-11-21T01:57:54,217 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35269 connecting to ZooKeeper ensemble=127.0.0.1:57707 2024-11-21T01:57:54,217 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:54,217 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:54,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:352690x0, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T01:57:54,221 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35269-0x101130399150003 connected 2024-11-21T01:57:54,221 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:35269-0x101130399150003, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-21T01:57:54,221 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-21T01:57:54,222 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T01:57:54,225 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/bucket.cache 2024-11-21T01:57:54,225 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T01:57:54,226 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T01:57:54,226 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T01:57:54,228 INFO [Thread-206 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T01:57:54,228 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T01:57:54,229 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T01:57:54,229 INFO [Thread-206 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-21T01:57:54,229 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:35269-0x101130399150003, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T01:57:54,231 INFO [Thread-206 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-21T01:57:54,231 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35269-0x101130399150003, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T01:57:54,233 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35269 2024-11-21T01:57:54,233 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35269 2024-11-21T01:57:54,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35269 2024-11-21T01:57:54,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35269 2024-11-21T01:57:54,234 DEBUG [Thread-206 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-21T01:57:54,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35269 2024-11-21T01:57:54,235 INFO [Thread-206 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T01:57:54,235 INFO [Thread-206 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-21T01:57:54,235 INFO [Thread-206 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/bucket.persistence is complete. 2024-11-21T01:57:54,236 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T01:57:54,236 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.HRegionServer(746): ClusterId : 6ec3dcf7-8e33-4c1a-a647-c87c490166cd 2024-11-21T01:57:54,236 DEBUG [RS:2;b93ada835ef8:35269 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T01:57:54,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51205, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T01:57:54,239 DEBUG [RS:2;b93ada835ef8:35269 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T01:57:54,239 DEBUG [RS:2;b93ada835ef8:35269 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T01:57:54,240 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T01:57:54,240 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T01:57:54,240 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T01:57:54,241 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=b93ada835ef8%2C40339%2C1732154270962.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,40339,1732154270962, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/oldWALs, maxLogs=55 2024-11-21T01:57:54,241 DEBUG [RS:2;b93ada835ef8:35269 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T01:57:54,242 DEBUG [RS:2;b93ada835ef8:35269 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@394d3aca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b93ada835ef8/172.17.0.3:0 2024-11-21T01:57:54,254 DEBUG [RS:2;b93ada835ef8:35269 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;b93ada835ef8:35269 2024-11-21T01:57:54,254 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T01:57:54,254 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T01:57:54,254 DEBUG [RS:2;b93ada835ef8:35269 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T01:57:54,254 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.HRegionServer(2659): reportForDuty to master=b93ada835ef8,42119,1732154270848 with port=35269, startcode=1732154274212 2024-11-21T01:57:54,255 DEBUG [RS:2;b93ada835ef8:35269 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T01:57:54,256 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38397, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T01:57:54,257 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42119 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b93ada835ef8,35269,1732154274212 2024-11-21T01:57:54,257 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42119 {}] master.ServerManager(517): Registering regionserver=b93ada835ef8,35269,1732154274212 2024-11-21T01:57:54,258 DEBUG [RS:2;b93ada835ef8:35269 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060 2024-11-21T01:57:54,259 DEBUG [RS:2;b93ada835ef8:35269 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T01:57:54,259 DEBUG [RS:2;b93ada835ef8:35269 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T01:57:54,259 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,40339,1732154270962/b93ada835ef8%2C40339%2C1732154270962.meta.1732154274242.meta 2024-11-21T01:57:54,259 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T01:57:54,260 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:57:54,260 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T01:57:54,260 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T01:57:54,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T01:57:54,260 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T01:57:54,260 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T01:57:54,260 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:57:54,261 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T01:57:54,261 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T01:57:54,261 DEBUG [RS:2;b93ada835ef8:35269 {}] zookeeper.ZKUtil(111): regionserver:35269-0x101130399150003, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b93ada835ef8,35269,1732154274212 2024-11-21T01:57:54,261 WARN [RS:2;b93ada835ef8:35269 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T01:57:54,261 INFO [RS:2;b93ada835ef8:35269 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T01:57:54,261 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b93ada835ef8,35269,1732154274212] 2024-11-21T01:57:54,261 DEBUG [RS:2;b93ada835ef8:35269 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,35269,1732154274212 2024-11-21T01:57:54,261 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T01:57:54,262 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T01:57:54,262 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:54,265 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/info/e16e834639254555ae1e0445bb27ee72, delay=978 ms 2024-11-21T01:57:54,266 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T01:57:54,266 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/info/e16e834639254555ae1e0445bb27ee72 2024-11-21T01:57:54,267 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:54,267 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T01:57:54,267 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T01:57:54,267 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:54,269 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T01:57:54,269 INFO [RS:2;b93ada835ef8:35269 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T01:57:54,269 INFO [RS:2;b93ada835ef8:35269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:54,269 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/ns/54d3cc8db22543569f9c9d4f43926cea, delay=985 ms 2024-11-21T01:57:54,269 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T01:57:54,270 INFO [RS:2;b93ada835ef8:35269 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T01:57:54,270 INFO [RS:2;b93ada835ef8:35269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:54,270 DEBUG [RS:2;b93ada835ef8:35269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:54,270 DEBUG [RS:2;b93ada835ef8:35269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:54,270 DEBUG [RS:2;b93ada835ef8:35269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:54,270 DEBUG [RS:2;b93ada835ef8:35269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:54,270 DEBUG [RS:2;b93ada835ef8:35269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:54,270 DEBUG [RS:2;b93ada835ef8:35269 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b93ada835ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-21T01:57:54,270 DEBUG [RS:2;b93ada835ef8:35269 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:54,270 DEBUG [RS:2;b93ada835ef8:35269 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:54,270 DEBUG [RS:2;b93ada835ef8:35269 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:54,270 DEBUG [RS:2;b93ada835ef8:35269 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:54,270 DEBUG [RS:2;b93ada835ef8:35269 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:54,270 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/ns/54d3cc8db22543569f9c9d4f43926cea 2024-11-21T01:57:54,270 DEBUG [RS:2;b93ada835ef8:35269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b93ada835ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-21T01:57:54,270 DEBUG [RS:2;b93ada835ef8:35269 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b93ada835ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-21T01:57:54,270 DEBUG [RS:2;b93ada835ef8:35269 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b93ada835ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-21T01:57:54,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:54,271 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T01:57:54,271 INFO [RS:2;b93ada835ef8:35269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:54,271 INFO [RS:2;b93ada835ef8:35269 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:54,271 INFO [RS:2;b93ada835ef8:35269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:54,271 INFO [RS:2;b93ada835ef8:35269 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:54,271 INFO [RS:2;b93ada835ef8:35269 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:54,271 INFO [RS:2;b93ada835ef8:35269 {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,35269,1732154274212-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T01:57:54,271 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T01:57:54,271 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:54,272 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:54,272 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T01:57:54,272 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T01:57:54,272 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:54,274 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/table/98a02fe0e9cc420aa364db87e5562208, delay=969 ms 2024-11-21T01:57:54,275 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/table/98a02fe0e9cc420aa364db87e5562208 2024-11-21T01:57:54,275 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T01:57:54,275 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T01:57:54,276 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740 2024-11-21T01:57:54,276 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740 2024-11-21T01:57:54,277 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T01:57:54,277 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T01:57:54,278 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T01:57:54,279 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T01:57:54,280 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66464716, jitterRate=-0.00959855318069458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T01:57:54,280 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T01:57:54,281 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732154274261Writing region info on filesystem at 1732154274261Initializing all the Stores at 1732154274261Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154274261Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154274261Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732154274261Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732154274261Cleaning up temporary data from old regions at 1732154274277 (+16 ms)Running coprocessor post-open hooks at 1732154274280 (+3 ms)Region opened successfully at 1732154274281 (+1 ms) 2024-11-21T01:57:54,282 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732154274235 2024-11-21T01:57:54,284 DEBUG [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T01:57:54,284 INFO [RS_OPEN_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T01:57:54,285 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=b93ada835ef8,40339,1732154270962 2024-11-21T01:57:54,286 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b93ada835ef8,40339,1732154270962, state=OPEN 2024-11-21T01:57:54,287 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T01:57:54,287 INFO [RS:2;b93ada835ef8:35269 {}] hbase.ChoreService(168): Chore ScheduledChore name=b93ada835ef8,35269,1732154274212-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:54,287 INFO [RS:2;b93ada835ef8:35269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:54,287 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.Replication(171): b93ada835ef8,35269,1732154274212 started 2024-11-21T01:57:54,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T01:57:54,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35269-0x101130399150003, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T01:57:54,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T01:57:54,288 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:54,288 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:54,288 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b93ada835ef8,40339,1732154270962 2024-11-21T01:57:54,288 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T01:57:54,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-21T01:57:54,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b93ada835ef8,40339,1732154270962 in 205 msec 2024-11-21T01:57:54,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T01:57:54,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 364 msec 2024-11-21T01:57:54,293 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): b93ada835ef8,44929,1732154270901 had 2 regions 2024-11-21T01:57:54,294 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure b93ada835ef8,44929,1732154270901, splitWal=true, meta=true, isMeta: false 2024-11-21T01:57:54,294 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,44929,1732154270901-splitting dir is empty, no logs to split. 2024-11-21T01:57:54,294 INFO [PEWorker-3 {}] master.SplitWALManager(105): b93ada835ef8,44929,1732154270901 WAL count=0, meta=false 2024-11-21T01:57:54,295 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,44929,1732154270901-splitting dir is empty, no logs to split. 2024-11-21T01:57:54,295 INFO [PEWorker-3 {}] master.SplitWALManager(105): b93ada835ef8,44929,1732154270901 WAL count=0, meta=false 2024-11-21T01:57:54,295 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if b93ada835ef8,44929,1732154270901 WAL splitting is done? wals=0, meta=false 2024-11-21T01:57:54,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732154272004, region=7c660ebcf5932b16019bde5cc2d9649d, ASSIGN}] 2024-11-21T01:57:54,298 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732154272004, region=7c660ebcf5932b16019bde5cc2d9649d, ASSIGN 2024-11-21T01:57:54,298 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732154272004, region=7c660ebcf5932b16019bde5cc2d9649d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T01:57:54,303 INFO [RS:2;b93ada835ef8:35269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T01:57:54,303 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.HRegionServer(1482): Serving as b93ada835ef8,35269,1732154274212, RpcServer on b93ada835ef8/172.17.0.3:35269, sessionid=0x101130399150003 2024-11-21T01:57:54,303 DEBUG [RS:2;b93ada835ef8:35269 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T01:57:54,303 DEBUG [RS:2;b93ada835ef8:35269 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b93ada835ef8,35269,1732154274212 2024-11-21T01:57:54,303 DEBUG [RS:2;b93ada835ef8:35269 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b93ada835ef8,35269,1732154274212' 2024-11-21T01:57:54,303 DEBUG [RS:2;b93ada835ef8:35269 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T01:57:54,304 DEBUG [RS:2;b93ada835ef8:35269 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T01:57:54,305 DEBUG [RS:2;b93ada835ef8:35269 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T01:57:54,305 DEBUG [RS:2;b93ada835ef8:35269 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T01:57:54,305 DEBUG [RS:2;b93ada835ef8:35269 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b93ada835ef8,35269,1732154274212 2024-11-21T01:57:54,305 DEBUG [RS:2;b93ada835ef8:35269 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b93ada835ef8,35269,1732154274212' 2024-11-21T01:57:54,305 DEBUG [RS:2;b93ada835ef8:35269 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T01:57:54,305 DEBUG [RS:2;b93ada835ef8:35269 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T01:57:54,305 DEBUG [RS:2;b93ada835ef8:35269 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T01:57:54,305 INFO [RS:2;b93ada835ef8:35269 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T01:57:54,306 INFO [RS:2;b93ada835ef8:35269 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T01:57:54,406 INFO [RS:2;b93ada835ef8:35269 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T01:57:54,407 INFO [RS:2;b93ada835ef8:35269 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=b93ada835ef8%2C35269%2C1732154274212, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,35269,1732154274212, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/oldWALs, maxLogs=55 2024-11-21T01:57:54,426 INFO [RS:2;b93ada835ef8:35269 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/WALs/b93ada835ef8,35269,1732154274212/b93ada835ef8%2C35269%2C1732154274212.1732154274408 2024-11-21T01:57:54,426 DEBUG [RS:2;b93ada835ef8:35269 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T01:57:54,449 DEBUG [b93ada835ef8:42119 {}] balancer.BalancerClusterState(204): Hosts are {b93ada835ef8=0} racks are {/default-rack=0} 2024-11-21T01:57:54,449 DEBUG [b93ada835ef8:42119 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T01:57:54,449 DEBUG [b93ada835ef8:42119 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T01:57:54,449 DEBUG [b93ada835ef8:42119 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T01:57:54,449 DEBUG [b93ada835ef8:42119 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T01:57:54,449 INFO [b93ada835ef8:42119 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T01:57:54,449 INFO [b93ada835ef8:42119 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T01:57:54,449 DEBUG [b93ada835ef8:42119 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T01:57:54,450 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=7c660ebcf5932b16019bde5cc2d9649d, regionState=OPENING, regionLocation=b93ada835ef8,40339,1732154270962 2024-11-21T01:57:54,451 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b93ada835ef8:44929 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b93ada835ef8/172.17.0.3:44929 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:57:54,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,44929,1732154270901, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,44929,1732154270901, seqNum=-1, error=java.net.ConnectException: Call to address=b93ada835ef8:44929 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b93ada835ef8/172.17.0.3:44929 2024-11-21T01:57:54,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,44929,1732154270901, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-21T01:57:54,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,44929,1732154270901, seqNum=-1 from cache 2024-11-21T01:57:54,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address b93ada835ef8:44929 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b93ada835ef8/172.17.0.3:44929 2024-11-21T01:57:54,559 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T01:57:54,559 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b93ada835ef8,40339,1732154270962, seqNum=-1] 2024-11-21T01:57:54,560 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T01:57:54,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36029, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T01:57:54,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732154272004, region=7c660ebcf5932b16019bde5cc2d9649d, ASSIGN because future has completed 2024-11-21T01:57:54,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c660ebcf5932b16019bde5cc2d9649d, server=b93ada835ef8,40339,1732154270962}] 2024-11-21T01:57:54,718 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:54,718 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c660ebcf5932b16019bde5cc2d9649d, NAME => 'testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d.', STARTKEY => '', ENDKEY => ''} 2024-11-21T01:57:54,719 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732154272004 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:54,719 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T01:57:54,719 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:54,719 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:54,720 INFO [StoreOpener-7c660ebcf5932b16019bde5cc2d9649d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:54,721 INFO [StoreOpener-7c660ebcf5932b16019bde5cc2d9649d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c660ebcf5932b16019bde5cc2d9649d columnFamilyName family 2024-11-21T01:57:54,721 DEBUG [StoreOpener-7c660ebcf5932b16019bde5cc2d9649d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T01:57:54,722 DEBUG [StoreFileOpener-7c660ebcf5932b16019bde5cc2d9649d-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/family/5fe9c79b4892475c8ca4222f4a1aa903, delay=977 ms 2024-11-21T01:57:54,724 DEBUG [StoreOpener-7c660ebcf5932b16019bde5cc2d9649d-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/family/5fe9c79b4892475c8ca4222f4a1aa903 2024-11-21T01:57:54,724 INFO [StoreOpener-7c660ebcf5932b16019bde5cc2d9649d-1 {}] regionserver.HStore(327): Store=7c660ebcf5932b16019bde5cc2d9649d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T01:57:54,724 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:54,725 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:54,725 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:54,725 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:54,725 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:54,726 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:54,727 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 7c660ebcf5932b16019bde5cc2d9649d; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60579082, jitterRate=-0.09730133414268494}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T01:57:54,727 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:54,728 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 7c660ebcf5932b16019bde5cc2d9649d: Running coprocessor pre-open hook at 1732154274719Writing region info on filesystem at 1732154274719Initializing all the Stores at 1732154274720 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732154274720Cleaning up temporary data from old regions at 1732154274725 (+5 ms)Running coprocessor post-open hooks at 1732154274727 (+2 ms)Region opened successfully at 1732154274728 (+1 ms) 2024-11-21T01:57:54,729 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d., pid=11, masterSystemTime=1732154274716 2024-11-21T01:57:54,731 DEBUG [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:54,731 INFO [RS_OPEN_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:54,732 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=7c660ebcf5932b16019bde5cc2d9649d, regionState=OPEN, openSeqNum=10, regionLocation=b93ada835ef8,40339,1732154270962 2024-11-21T01:57:54,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c660ebcf5932b16019bde5cc2d9649d, server=b93ada835ef8,40339,1732154270962 because future has completed 2024-11-21T01:57:54,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-21T01:57:54,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c660ebcf5932b16019bde5cc2d9649d, server=b93ada835ef8,40339,1732154270962 in 175 msec 2024-11-21T01:57:54,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-21T01:57:54,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732154272004, region=7c660ebcf5932b16019bde5cc2d9649d, ASSIGN in 444 msec 2024-11-21T01:57:54,743 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server b93ada835ef8,44929,1732154270901 after splitting done 2024-11-21T01:57:54,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure b93ada835ef8,44929,1732154270901, splitWal=true, meta=true in 827 msec 2024-11-21T01:57:54,803 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T01:57:54,804 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T01:57:54,804 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:57:54,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:54,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:54,804 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T01:57:54,804 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T01:57:54,804 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1193834594, stopped=false 2024-11-21T01:57:54,804 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b93ada835ef8,42119,1732154270848 2024-11-21T01:57:54,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T01:57:54,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T01:57:54,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35269-0x101130399150003, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T01:57:54,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:54,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:54,806 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T01:57:54,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35269-0x101130399150003, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:54,806 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T01:57:54,806 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:57:54,806 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:54,807 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b93ada835ef8,40339,1732154270962' ***** 2024-11-21T01:57:54,807 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35269-0x101130399150003, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T01:57:54,807 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T01:57:54,807 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b93ada835ef8,35269,1732154274212' ***** 2024-11-21T01:57:54,807 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T01:57:54,807 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T01:57:54,807 INFO [RS:1;b93ada835ef8:40339 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T01:57:54,807 INFO [RS:2;b93ada835ef8:35269 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T01:57:54,807 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T01:57:54,807 INFO [RS:1;b93ada835ef8:40339 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/bucket.persistence 2024-11-21T01:57:54,807 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T01:57:54,807 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T01:57:54,807 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T01:57:54,807 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T01:57:54,808 INFO [RS:1;b93ada835ef8:40339 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T01:57:54,808 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T01:57:54,807 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T01:57:54,808 INFO [RS:2;b93ada835ef8:35269 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/29a1133b-93ad-a702-dd70-c2e8028a3fae/bucket.persistence 2024-11-21T01:57:54,808 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T01:57:54,808 INFO [RS:2;b93ada835ef8:35269 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T01:57:54,808 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T01:57:54,818 DEBUG [RS:1;b93ada835ef8:40339 {}] bucket.BucketCache(1487): Thread RS:1;b93ada835ef8:40339 started persisting bucket cache to file 2024-11-21T01:57:54,818 DEBUG [RS:1;b93ada835ef8:40339 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T01:57:54,818 DEBUG [RS:1;b93ada835ef8:40339 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T01:57:54,818 DEBUG [RS:2;b93ada835ef8:35269 {}] bucket.BucketCache(1487): Thread RS:2;b93ada835ef8:35269 started persisting bucket cache to file 2024-11-21T01:57:54,819 DEBUG [RS:2;b93ada835ef8:35269 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T01:57:54,819 DEBUG [RS:2;b93ada835ef8:35269 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T01:57:54,821 DEBUG [RS:1;b93ada835ef8:40339 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T01:57:54,821 DEBUG [RS:1;b93ada835ef8:40339 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732154274818 2024-11-21T01:57:54,822 DEBUG [RS:1;b93ada835ef8:40339 {}] bucket.BucketCache(1509): Thread RS:1;b93ada835ef8:40339 finished persisting bucket cache to file, renaming 2024-11-21T01:57:54,822 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T01:57:54,822 INFO [RS:1;b93ada835ef8:40339 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T01:57:54,822 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T01:57:54,822 INFO [RS:1;b93ada835ef8:40339 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T01:57:54,821 ERROR [RS:2;b93ada835ef8:35269 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1056) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T01:57:54,822 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer(3091): Received CLOSE for 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:54,822 DEBUG [RS:2;b93ada835ef8:35269 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-21T01:57:54,822 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer(959): stopping server b93ada835ef8,40339,1732154270962 2024-11-21T01:57:54,822 DEBUG [RS:2;b93ada835ef8:35269 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732154274818 2024-11-21T01:57:54,822 INFO [RS:1;b93ada835ef8:40339 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T01:57:54,822 DEBUG [RS:2;b93ada835ef8:35269 {}] bucket.BucketCache(1509): Thread RS:2;b93ada835ef8:35269 finished persisting bucket cache to file, renaming 2024-11-21T01:57:54,822 INFO [RS:1;b93ada835ef8:40339 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;b93ada835ef8:40339. 2024-11-21T01:57:54,822 WARN [RS:2;b93ada835ef8:35269 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-21T01:57:54,822 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T01:57:54,822 DEBUG [RS:1;b93ada835ef8:40339 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:57:54,822 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7c660ebcf5932b16019bde5cc2d9649d, disabling compactions & flushes 2024-11-21T01:57:54,822 DEBUG [RS:1;b93ada835ef8:40339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:54,822 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T01:57:54,823 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:54,823 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:54,823 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T01:57:54,823 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T01:57:54,823 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. after waiting 0 ms 2024-11-21T01:57:54,823 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T01:57:54,823 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T01:57:54,823 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:54,823 INFO [RS:2;b93ada835ef8:35269 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T01:57:54,823 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T01:57:54,823 INFO [RS:2;b93ada835ef8:35269 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T01:57:54,823 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.HRegionServer(959): stopping server b93ada835ef8,35269,1732154274212 2024-11-21T01:57:54,823 INFO [RS:2;b93ada835ef8:35269 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T01:57:54,823 INFO [RS:2;b93ada835ef8:35269 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;b93ada835ef8:35269. 2024-11-21T01:57:54,823 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-21T01:57:54,823 DEBUG [RS:2;b93ada835ef8:35269 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T01:57:54,823 DEBUG [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 7c660ebcf5932b16019bde5cc2d9649d=testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d.} 2024-11-21T01:57:54,823 DEBUG [RS:2;b93ada835ef8:35269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:54,823 DEBUG [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7c660ebcf5932b16019bde5cc2d9649d 2024-11-21T01:57:54,823 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T01:57:54,823 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.HRegionServer(976): stopping server b93ada835ef8,35269,1732154274212; all regions closed. 2024-11-21T01:57:54,823 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T01:57:54,823 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T01:57:54,824 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T01:57:54,824 DEBUG [StoreFileCloser-7c660ebcf5932b16019bde5cc2d9649d-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/family/5fe9c79b4892475c8ca4222f4a1aa903 2024-11-21T01:57:54,824 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T01:57:54,824 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T01:57:54,824 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-21T01:57:54,825 DEBUG [RS:2;b93ada835ef8:35269 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/oldWALs 2024-11-21T01:57:54,825 INFO [RS:2;b93ada835ef8:35269 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b93ada835ef8%2C35269%2C1732154274212:(num 1732154274408) 2024-11-21T01:57:54,825 DEBUG [RS:2;b93ada835ef8:35269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:54,825 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T01:57:54,825 INFO [RS:2;b93ada835ef8:35269 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T01:57:54,825 INFO [RS:2;b93ada835ef8:35269 {}] hbase.ChoreService(370): Chore service for: regionserver/b93ada835ef8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T01:57:54,826 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T01:57:54,826 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T01:57:54,826 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T01:57:54,826 INFO [regionserver/b93ada835ef8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T01:57:54,826 INFO [RS:2;b93ada835ef8:35269 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T01:57:54,826 INFO [RS:2;b93ada835ef8:35269 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35269 2024-11-21T01:57:54,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T01:57:54,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35269-0x101130399150003, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b93ada835ef8,35269,1732154274212 2024-11-21T01:57:54,828 INFO [RS:2;b93ada835ef8:35269 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T01:57:54,829 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/default/testBlockEvictionOnGracefulStop1732154272004/7c660ebcf5932b16019bde5cc2d9649d/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-21T01:57:54,829 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b93ada835ef8,35269,1732154274212] 2024-11-21T01:57:54,829 INFO [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:54,829 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7c660ebcf5932b16019bde5cc2d9649d: Waiting for close lock at 1732154274822Running coprocessor pre-close hooks at 1732154274822Disabling compacts and flushes for region at 1732154274822Disabling writes for close at 1732154274823 (+1 ms)Writing region close event to WAL at 1732154274824 (+1 ms)Running coprocessor post-close hooks at 1732154274829 (+5 ms)Closed at 1732154274829 2024-11-21T01:57:54,829 DEBUG [RS_CLOSE_REGION-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d. 2024-11-21T01:57:54,830 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b93ada835ef8,35269,1732154274212 already deleted, retry=false 2024-11-21T01:57:54,831 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b93ada835ef8,35269,1732154274212 expired; onlineServers=1 2024-11-21T01:57:54,836 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 367f535f3c1742398e2d9ea8a43e0d9b 2024-11-21T01:57:54,850 INFO [regionserver/b93ada835ef8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T01:57:54,850 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/info/367f535f3c1742398e2d9ea8a43e0d9b is 209, key is testBlockEvictionOnGracefulStop1732154272004,,1732154272008.7c660ebcf5932b16019bde5cc2d9649d./info:regioninfo/1732154274731/Put/seqid=0 2024-11-21T01:57:54,850 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/info/367f535f3c1742398e2d9ea8a43e0d9b 2024-11-21T01:57:54,853 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/.tmp/info/367f535f3c1742398e2d9ea8a43e0d9b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/info/367f535f3c1742398e2d9ea8a43e0d9b 2024-11-21T01:57:54,854 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/info/367f535f3c1742398e2d9ea8a43e0d9b, delay=967 ms 2024-11-21T01:57:54,855 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/info/367f535f3c1742398e2d9ea8a43e0d9b, entries=8, sequenceid=19, filesize=6.6 K 2024-11-21T01:57:54,855 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false 2024-11-21T01:57:54,856 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/info/e16e834639254555ae1e0445bb27ee72 2024-11-21T01:57:54,856 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/info/367f535f3c1742398e2d9ea8a43e0d9b 2024-11-21T01:57:54,857 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/ns/54d3cc8db22543569f9c9d4f43926cea 2024-11-21T01:57:54,857 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/table/98a02fe0e9cc420aa364db87e5562208 2024-11-21T01:57:54,861 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-21T01:57:54,861 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T01:57:54,861 INFO [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T01:57:54,862 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732154274823Running coprocessor pre-close hooks at 1732154274823Disabling compacts and flushes for region at 1732154274823Disabling writes for close at 1732154274824 (+1 ms)Obtaining lock to block concurrent updates at 1732154274824Preparing flush snapshotting stores in 1588230740 at 1732154274824Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732154274824Flushing stores of hbase:meta,,1.1588230740 at 1732154274825 (+1 ms)Flushing 1588230740/info: creating writer at 1732154274825Flushing 1588230740/info: appending metadata at 1732154274849 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732154274849Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e4f3ade: reopening flushed file at 1732154274852 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false at 1732154274855 (+3 ms)Writing region close event to WAL at 1732154274857 (+2 ms)Running coprocessor post-close hooks at 1732154274861 (+4 ms)Closed at 1732154274861 2024-11-21T01:57:54,862 DEBUG [RS_CLOSE_META-regionserver/b93ada835ef8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T01:57:54,872 INFO [regionserver/b93ada835ef8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T01:57:54,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35269-0x101130399150003, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:57:54,930 INFO [RS:2;b93ada835ef8:35269 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T01:57:54,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35269-0x101130399150003, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:57:54,930 INFO [RS:2;b93ada835ef8:35269 {}] regionserver.HRegionServer(1031): Exiting; stopping=b93ada835ef8,35269,1732154274212; zookeeper connection closed. 2024-11-21T01:57:54,930 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4427bc0c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4427bc0c 2024-11-21T01:57:55,023 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer(976): stopping server b93ada835ef8,40339,1732154270962; all regions closed. 2024-11-21T01:57:55,026 DEBUG [RS:1;b93ada835ef8:40339 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/oldWALs 2024-11-21T01:57:55,026 INFO [RS:1;b93ada835ef8:40339 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b93ada835ef8%2C40339%2C1732154270962.meta:.meta(num 1732154274242) 2024-11-21T01:57:55,027 DEBUG [RS:1;b93ada835ef8:40339 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/oldWALs 2024-11-21T01:57:55,027 INFO [RS:1;b93ada835ef8:40339 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b93ada835ef8%2C40339%2C1732154270962:(num 1732154271493) 2024-11-21T01:57:55,027 DEBUG [RS:1;b93ada835ef8:40339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T01:57:55,027 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T01:57:55,027 INFO [RS:1;b93ada835ef8:40339 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T01:57:55,028 INFO [RS:1;b93ada835ef8:40339 {}] hbase.ChoreService(370): Chore service for: regionserver/b93ada835ef8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T01:57:55,028 INFO [RS:1;b93ada835ef8:40339 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T01:57:55,028 INFO [regionserver/b93ada835ef8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T01:57:55,028 INFO [RS:1;b93ada835ef8:40339 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40339 2024-11-21T01:57:55,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T01:57:55,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b93ada835ef8,40339,1732154270962 2024-11-21T01:57:55,031 INFO [RS:1;b93ada835ef8:40339 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T01:57:55,032 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b93ada835ef8,40339,1732154270962] 2024-11-21T01:57:55,033 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b93ada835ef8,40339,1732154270962 already deleted, retry=false 2024-11-21T01:57:55,033 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b93ada835ef8,40339,1732154270962 expired; onlineServers=0 2024-11-21T01:57:55,034 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b93ada835ef8,42119,1732154270848' ***** 2024-11-21T01:57:55,034 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T01:57:55,034 INFO [M:0;b93ada835ef8:42119 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T01:57:55,034 INFO [M:0;b93ada835ef8:42119 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T01:57:55,034 DEBUG [M:0;b93ada835ef8:42119 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T01:57:55,034 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T01:57:55,034 DEBUG [M:0;b93ada835ef8:42119 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T01:57:55,034 DEBUG [master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.large.0-1732154271226 {}] cleaner.HFileCleaner(306): Exit Thread[master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.large.0-1732154271226,5,FailOnTimeoutGroup] 2024-11-21T01:57:55,034 DEBUG [master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.small.0-1732154271226 {}] cleaner.HFileCleaner(306): Exit Thread[master/b93ada835ef8:0:becomeActiveMaster-HFileCleaner.small.0-1732154271226,5,FailOnTimeoutGroup] 2024-11-21T01:57:55,034 INFO [M:0;b93ada835ef8:42119 {}] hbase.ChoreService(370): Chore service for: master/b93ada835ef8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T01:57:55,034 INFO [M:0;b93ada835ef8:42119 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T01:57:55,034 DEBUG [M:0;b93ada835ef8:42119 {}] master.HMaster(1795): Stopping service threads 2024-11-21T01:57:55,034 INFO [M:0;b93ada835ef8:42119 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T01:57:55,035 INFO [M:0;b93ada835ef8:42119 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T01:57:55,035 INFO [M:0;b93ada835ef8:42119 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T01:57:55,035 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T01:57:55,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T01:57:55,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T01:57:55,036 DEBUG [M:0;b93ada835ef8:42119 {}] zookeeper.ZKUtil(347): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T01:57:55,036 WARN [M:0;b93ada835ef8:42119 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T01:57:55,036 INFO [M:0;b93ada835ef8:42119 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/.lastflushedseqids 2024-11-21T01:57:55,043 INFO [M:0;b93ada835ef8:42119 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T01:57:55,044 INFO [M:0;b93ada835ef8:42119 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T01:57:55,044 DEBUG [M:0;b93ada835ef8:42119 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T01:57:55,044 INFO [M:0;b93ada835ef8:42119 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:57:55,044 DEBUG [M:0;b93ada835ef8:42119 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:57:55,044 DEBUG [M:0;b93ada835ef8:42119 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T01:57:55,044 DEBUG [M:0;b93ada835ef8:42119 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:57:55,044 INFO [M:0;b93ada835ef8:42119 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-21T01:57:55,057 DEBUG [M:0;b93ada835ef8:42119 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c3061c69d9c846f3a53cc989708c2b9d 2024-11-21T01:57:55,072 DEBUG [M:0;b93ada835ef8:42119 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c3061c69d9c846f3a53cc989708c2b9d is 82, key is hbase:meta,,1/info:regioninfo/1732154274285/Put/seqid=0 2024-11-21T01:57:55,073 INFO [M:0;b93ada835ef8:42119 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c3061c69d9c846f3a53cc989708c2b9d 2024-11-21T01:57:55,081 DEBUG [M:0;b93ada835ef8:42119 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2f92317fb7304317b7ee851bfc21ba5a 2024-11-21T01:57:55,096 DEBUG [M:0;b93ada835ef8:42119 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2f92317fb7304317b7ee851bfc21ba5a is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732154272385/Put/seqid=0 2024-11-21T01:57:55,096 INFO [M:0;b93ada835ef8:42119 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2f92317fb7304317b7ee851bfc21ba5a 2024-11-21T01:57:55,111 DEBUG [M:0;b93ada835ef8:42119 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 23b914d3b1544be782653cfcbfb40ee1 2024-11-21T01:57:55,127 DEBUG [M:0;b93ada835ef8:42119 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/23b914d3b1544be782653cfcbfb40ee1 is 69, key is b93ada835ef8,35269,1732154274212/rs:state/1732154274257/Put/seqid=0 2024-11-21T01:57:55,128 INFO [M:0;b93ada835ef8:42119 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/23b914d3b1544be782653cfcbfb40ee1 2024-11-21T01:57:55,130 INFO [M:0;b93ada835ef8:42119 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 23b914d3b1544be782653cfcbfb40ee1 2024-11-21T01:57:55,130 DEBUG [M:0;b93ada835ef8:42119 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c3061c69d9c846f3a53cc989708c2b9d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c3061c69d9c846f3a53cc989708c2b9d 2024-11-21T01:57:55,132 INFO [M:0;b93ada835ef8:42119 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c3061c69d9c846f3a53cc989708c2b9d, entries=14, sequenceid=102, filesize=5.9 K 2024-11-21T01:57:55,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:57:55,132 DEBUG [M:0;b93ada835ef8:42119 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2f92317fb7304317b7ee851bfc21ba5a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2f92317fb7304317b7ee851bfc21ba5a 2024-11-21T01:57:55,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40339-0x101130399150002, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:57:55,132 INFO [RS:1;b93ada835ef8:40339 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T01:57:55,133 INFO [RS:1;b93ada835ef8:40339 {}] regionserver.HRegionServer(1031): Exiting; stopping=b93ada835ef8,40339,1732154270962; zookeeper connection closed. 2024-11-21T01:57:55,133 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1e719195 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1e719195 2024-11-21T01:57:55,133 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-21T01:57:55,134 INFO [M:0;b93ada835ef8:42119 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2f92317fb7304317b7ee851bfc21ba5a, entries=11, sequenceid=102, filesize=6.5 K 2024-11-21T01:57:55,134 DEBUG [M:0;b93ada835ef8:42119 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/23b914d3b1544be782653cfcbfb40ee1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/23b914d3b1544be782653cfcbfb40ee1 2024-11-21T01:57:55,136 INFO [M:0;b93ada835ef8:42119 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 23b914d3b1544be782653cfcbfb40ee1 2024-11-21T01:57:55,136 INFO [M:0;b93ada835ef8:42119 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0a621e60-6799-b6a6-f78e-fe7b7428998c/ed989d45-81c3-0928-cc38-9433497aa060/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/23b914d3b1544be782653cfcbfb40ee1, entries=3, sequenceid=102, filesize=5.3 K 2024-11-21T01:57:55,136 INFO [M:0;b93ada835ef8:42119 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=102, compaction requested=false 2024-11-21T01:57:55,138 INFO [M:0;b93ada835ef8:42119 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T01:57:55,138 DEBUG [M:0;b93ada835ef8:42119 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732154275044Disabling compacts and flushes for region at 1732154275044Disabling writes for close at 1732154275044Obtaining lock to block concurrent updates at 1732154275044Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732154275044Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1732154275044Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732154275045 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732154275045Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732154275071 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732154275072 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732154275075 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732154275095 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732154275095Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732154275098 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732154275126 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732154275126Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39b5d7b: reopening flushed file at 1732154275130 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1226c6c0: reopening flushed file at 1732154275132 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b46b127: reopening flushed file at 1732154275134 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=102, compaction requested=false at 1732154275137 (+3 ms)Writing region close event to WAL at 1732154275138 (+1 ms)Closed at 1732154275138 2024-11-21T01:57:55,139 INFO [M:0;b93ada835ef8:42119 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T01:57:55,139 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T01:57:55,139 INFO [M:0;b93ada835ef8:42119 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42119 2024-11-21T01:57:55,140 INFO [M:0;b93ada835ef8:42119 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T01:57:55,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:57:55,242 INFO [M:0;b93ada835ef8:42119 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T01:57:55,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42119-0x101130399150000, quorum=127.0.0.1:57707, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T01:57:55,245 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T01:57:55,247 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-21T01:57:55,247 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T01:57:55,255 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 49) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=726 (was 763), ProcessCount=11 (was 11), AvailableMemoryMB=2693 (was 2887)