2024-11-11 20:52:16,303 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-11 20:52:16,335 main DEBUG Took 0.027221 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 20:52:16,336 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 20:52:16,336 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 20:52:16,338 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 20:52:16,339 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:16,347 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 20:52:16,360 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:16,362 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:16,363 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:16,363 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:16,364 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:16,365 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:16,366 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:16,366 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:16,367 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:16,368 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:16,369 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:16,369 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:16,370 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:16,371 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:16,372 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:16,372 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:16,373 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:16,373 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:16,374 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:16,374 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:16,375 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:16,376 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:16,376 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:16,377 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:16,378 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:16,378 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 20:52:16,380 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:16,382 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 20:52:16,384 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 20:52:16,385 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 20:52:16,387 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 20:52:16,387 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 20:52:16,398 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 20:52:16,402 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 20:52:16,404 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 20:52:16,405 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 20:52:16,405 main DEBUG createAppenders(={Console}) 2024-11-11 20:52:16,406 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-11 20:52:16,407 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-11 20:52:16,407 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-11 20:52:16,408 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 20:52:16,408 main DEBUG OutputStream closed 2024-11-11 20:52:16,409 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 20:52:16,409 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 20:52:16,409 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-11 20:52:16,511 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 20:52:16,514 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 20:52:16,516 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 20:52:16,519 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 20:52:16,520 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 20:52:16,521 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 20:52:16,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 20:52:16,525 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 20:52:16,526 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 20:52:16,526 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 20:52:16,527 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 20:52:16,528 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 20:52:16,528 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 20:52:16,529 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 20:52:16,529 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 20:52:16,530 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 20:52:16,530 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 20:52:16,531 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 20:52:16,535 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 20:52:16,535 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-11 20:52:16,536 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 20:52:16,538 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-11T20:52:16,558 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-11 20:52:16,564 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 20:52:16,564 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T20:52:16,916 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f 2024-11-11T20:52:16,971 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=226, ProcessCount=11, AvailableMemoryMB=6199 2024-11-11T20:52:17,117 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T20:52:17,260 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:17,285 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/cluster_c960e880-f7f5-f7e4-372a-fdb3301ea10f, deleteOnExit=true 2024-11-11T20:52:17,438 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/cluster_c960e880-f7f5-f7e4-372a-fdb3301ea10f/zookeeper_0, clientPort=59135, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/cluster_c960e880-f7f5-f7e4-372a-fdb3301ea10f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/cluster_c960e880-f7f5-f7e4-372a-fdb3301ea10f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T20:52:17,459 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59135 2024-11-11T20:52:17,460 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:17,587 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8 with version=8 2024-11-11T20:52:17,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/hbase-staging 2024-11-11T20:52:17,695 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T20:52:18,042 INFO [Time-limited test {}] client.ConnectionUtils(128): master/51ca66f7ee3c:0 server-side Connection retries=45 2024-11-11T20:52:18,054 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:18,056 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:18,067 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T20:52:18,068 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:18,068 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T20:52:18,215 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T20:52:18,307 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T20:52:18,357 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T20:52:18,369 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T20:52:18,373 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T20:52:18,410 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 39851 (auto-detected) 2024-11-11T20:52:18,412 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-11T20:52:18,435 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39535 2024-11-11T20:52:18,457 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39535 connecting to ZooKeeper ensemble=127.0.0.1:59135 2024-11-11T20:52:18,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:395350x0, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T20:52:18,493 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39535-0x100308f05e40000 connected 2024-11-11T20:52:18,526 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:18,527 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:18,535 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:52:18,540 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8, hbase.cluster.distributed=false 2024-11-11T20:52:18,562 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T20:52:18,567 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39535 2024-11-11T20:52:18,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39535 2024-11-11T20:52:18,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39535 2024-11-11T20:52:18,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39535 2024-11-11T20:52:18,569 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39535 2024-11-11T20:52:18,644 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/51ca66f7ee3c:0 server-side Connection retries=45 2024-11-11T20:52:18,645 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:18,645 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:18,646 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T20:52:18,646 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:18,646 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T20:52:18,648 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T20:52:18,651 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T20:52:18,652 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42859 2024-11-11T20:52:18,654 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42859 connecting to ZooKeeper ensemble=127.0.0.1:59135 2024-11-11T20:52:18,655 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:18,655 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:18,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:428590x0, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T20:52:18,659 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42859-0x100308f05e40001 connected 2024-11-11T20:52:18,659 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:52:18,664 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T20:52:18,676 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/bucket.cache 2024-11-11T20:52:18,682 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-11T20:52:18,683 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-11T20:52:18,684 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-11T20:52:18,686 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-11T20:52:18,686 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-11T20:52:18,694 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-11T20:52:18,697 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-11T20:52:18,697 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/bucket.persistence is complete. 2024-11-11T20:52:18,700 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T20:52:18,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T20:52:18,708 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T20:52:18,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42859 2024-11-11T20:52:18,713 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42859 2024-11-11T20:52:18,714 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42859 2024-11-11T20:52:18,715 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42859 2024-11-11T20:52:18,715 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42859 2024-11-11T20:52:18,733 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/51ca66f7ee3c:0 server-side Connection retries=45 2024-11-11T20:52:18,733 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:18,733 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:18,733 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T20:52:18,734 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:18,734 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T20:52:18,734 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T20:52:18,734 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T20:52:18,736 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42089 2024-11-11T20:52:18,739 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42089 connecting to ZooKeeper ensemble=127.0.0.1:59135 2024-11-11T20:52:18,739 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:18,740 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:18,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:420890x0, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T20:52:18,745 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:420890x0, quorum=127.0.0.1:59135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:52:18,745 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42089-0x100308f05e40002 connected 2024-11-11T20:52:18,745 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T20:52:18,747 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/bucket.cache 2024-11-11T20:52:18,747 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-11T20:52:18,747 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-11T20:52:18,747 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-11T20:52:18,748 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-11T20:52:18,748 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-11T20:52:18,748 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-11T20:52:18,749 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T20:52:18,749 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-11T20:52:18,749 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/bucket.persistence is complete. 2024-11-11T20:52:18,750 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T20:52:18,752 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T20:52:18,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42089 2024-11-11T20:52:18,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42089 2024-11-11T20:52:18,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42089 2024-11-11T20:52:18,762 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42089 2024-11-11T20:52:18,762 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42089 2024-11-11T20:52:18,779 DEBUG [M:0;51ca66f7ee3c:39535 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;51ca66f7ee3c:39535 2024-11-11T20:52:18,781 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/51ca66f7ee3c,39535,1731358337804 2024-11-11T20:52:18,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:52:18,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:52:18,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:52:18,792 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/51ca66f7ee3c,39535,1731358337804 2024-11-11T20:52:18,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T20:52:18,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T20:52:18,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:18,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:18,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:18,822 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T20:52:18,823 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/51ca66f7ee3c,39535,1731358337804 from backup master directory 2024-11-11T20:52:18,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:52:18,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:52:18,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/51ca66f7ee3c,39535,1731358337804 2024-11-11T20:52:18,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:52:18,827 WARN [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T20:52:18,828 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=51ca66f7ee3c,39535,1731358337804 2024-11-11T20:52:18,830 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T20:52:18,831 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T20:52:18,841 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/hbase.id] with ID: 6c67f0b4-4791-4b2c-a567-2be657e06ec3 2024-11-11T20:52:18,842 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/.tmp/hbase.id 2024-11-11T20:52:18,847 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/hbase.id] 2024-11-11T20:52:18,889 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:18,896 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T20:52:18,918 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-11T20:52:18,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:18,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:18,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:18,951 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:52:18,955 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T20:52:18,996 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:19,002 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:52:19,054 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store 2024-11-11T20:52:19,077 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T20:52:19,082 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:19,084 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T20:52:19,084 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:19,085 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:19,087 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T20:52:19,087 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:19,087 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:19,088 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731358339084Disabling compacts and flushes for region at 1731358339084Disabling writes for close at 1731358339087 (+3 ms)Writing region close event to WAL at 1731358339087Closed at 1731358339087 2024-11-11T20:52:19,089 WARN [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/.initializing 2024-11-11T20:52:19,090 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/WALs/51ca66f7ee3c,39535,1731358337804 2024-11-11T20:52:19,104 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T20:52:19,120 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51ca66f7ee3c%2C39535%2C1731358337804, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/WALs/51ca66f7ee3c,39535,1731358337804, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/oldWALs, maxLogs=10 2024-11-11T20:52:19,159 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/WALs/51ca66f7ee3c,39535,1731358337804/51ca66f7ee3c%2C39535%2C1731358337804.1731358339126 2024-11-11T20:52:19,160 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T20:52:19,161 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:52:19,161 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:19,165 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:19,166 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:19,202 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:19,230 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T20:52:19,235 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:19,236 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:19,237 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:19,242 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T20:52:19,242 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:19,243 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:19,243 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:19,248 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T20:52:19,248 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:19,249 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:19,249 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:19,254 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T20:52:19,254 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:19,254 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:19,255 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:19,258 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:19,259 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:19,262 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:19,263 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:19,265 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T20:52:19,268 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:19,274 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:52:19,275 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71306608, jitterRate=0.06255125999450684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T20:52:19,283 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731358339179Initializing all the Stores at 1731358339180 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358339180Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358339181 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358339181Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358339182 (+1 ms)Cleaning up temporary data from old regions at 1731358339263 (+81 ms)Region opened successfully at 1731358339282 (+19 ms) 2024-11-11T20:52:19,284 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T20:52:19,317 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3288fea9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51ca66f7ee3c/172.17.0.2:0 2024-11-11T20:52:19,353 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T20:52:19,365 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T20:52:19,365 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T20:52:19,369 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:52:19,370 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:52:19,378 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-11T20:52:19,379 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T20:52:19,406 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T20:52:19,415 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T20:52:19,447 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T20:52:19,449 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T20:52:19,451 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T20:52:19,453 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T20:52:19,455 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T20:52:19,459 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T20:52:19,460 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T20:52:19,462 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T20:52:19,463 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T20:52:19,479 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T20:52:19,481 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T20:52:19,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T20:52:19,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T20:52:19,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T20:52:19,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:19,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:19,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:19,499 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=51ca66f7ee3c,39535,1731358337804, sessionid=0x100308f05e40000, setting cluster-up flag (Was=false) 2024-11-11T20:52:19,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:19,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:19,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:19,515 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T20:52:19,518 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=51ca66f7ee3c,39535,1731358337804 2024-11-11T20:52:19,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:19,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:19,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:19,527 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T20:52:19,529 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=51ca66f7ee3c,39535,1731358337804 2024-11-11T20:52:19,535 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T20:52:19,566 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.HRegionServer(746): ClusterId : 6c67f0b4-4791-4b2c-a567-2be657e06ec3 2024-11-11T20:52:19,566 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.HRegionServer(746): ClusterId : 6c67f0b4-4791-4b2c-a567-2be657e06ec3 2024-11-11T20:52:19,568 DEBUG [RS:1;51ca66f7ee3c:42089 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T20:52:19,568 DEBUG [RS:0;51ca66f7ee3c:42859 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T20:52:19,572 DEBUG [RS:1;51ca66f7ee3c:42089 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T20:52:19,572 DEBUG [RS:0;51ca66f7ee3c:42859 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T20:52:19,572 DEBUG [RS:1;51ca66f7ee3c:42089 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T20:52:19,572 DEBUG [RS:0;51ca66f7ee3c:42859 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T20:52:19,575 DEBUG [RS:1;51ca66f7ee3c:42089 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T20:52:19,575 DEBUG [RS:0;51ca66f7ee3c:42859 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T20:52:19,576 DEBUG [RS:1;51ca66f7ee3c:42089 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ca6a148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51ca66f7ee3c/172.17.0.2:0 2024-11-11T20:52:19,576 DEBUG [RS:0;51ca66f7ee3c:42859 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@687d750c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51ca66f7ee3c/172.17.0.2:0 2024-11-11T20:52:19,595 DEBUG [RS:1;51ca66f7ee3c:42089 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;51ca66f7ee3c:42089 2024-11-11T20:52:19,597 DEBUG [RS:0;51ca66f7ee3c:42859 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;51ca66f7ee3c:42859 2024-11-11T20:52:19,599 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T20:52:19,599 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T20:52:19,599 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T20:52:19,599 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T20:52:19,599 DEBUG [RS:0;51ca66f7ee3c:42859 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T20:52:19,599 DEBUG [RS:1;51ca66f7ee3c:42089 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T20:52:19,602 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.HRegionServer(2659): reportForDuty to master=51ca66f7ee3c,39535,1731358337804 with port=42859, startcode=1731358338612 2024-11-11T20:52:19,602 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.HRegionServer(2659): reportForDuty to master=51ca66f7ee3c,39535,1731358337804 with port=42089, startcode=1731358338732 2024-11-11T20:52:19,616 DEBUG [RS:0;51ca66f7ee3c:42859 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T20:52:19,616 DEBUG [RS:1;51ca66f7ee3c:42089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T20:52:19,616 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:52:19,630 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T20:52:19,639 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T20:52:19,645 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 51ca66f7ee3c,39535,1731358337804 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T20:52:19,654 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:52:19,655 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:52:19,655 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:52:19,655 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:52:19,655 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/51ca66f7ee3c:0, corePoolSize=10, maxPoolSize=10 2024-11-11T20:52:19,656 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,656 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=2, maxPoolSize=2 2024-11-11T20:52:19,656 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,666 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T20:52:19,666 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358369666 2024-11-11T20:52:19,666 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T20:52:19,668 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T20:52:19,669 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T20:52:19,672 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:19,673 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T20:52:19,673 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T20:52:19,673 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T20:52:19,674 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T20:52:19,674 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T20:52:19,676 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,685 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T20:52:19,687 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T20:52:19,687 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T20:52:19,689 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T20:52:19,689 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T20:52:19,693 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36837, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T20:52:19,693 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33185, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T20:52:19,693 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.large.0-1731358339691,5,FailOnTimeoutGroup] 2024-11-11T20:52:19,697 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.small.0-1731358339693,5,FailOnTimeoutGroup] 2024-11-11T20:52:19,697 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,697 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T20:52:19,698 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,699 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,701 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39535 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 51ca66f7ee3c,42089,1731358338732 2024-11-11T20:52:19,702 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T20:52:19,703 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8 2024-11-11T20:52:19,703 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39535 {}] master.ServerManager(517): Registering regionserver=51ca66f7ee3c,42089,1731358338732 2024-11-11T20:52:19,714 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39535 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 51ca66f7ee3c,42859,1731358338612 2024-11-11T20:52:19,714 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39535 {}] master.ServerManager(517): Registering regionserver=51ca66f7ee3c,42859,1731358338612 2024-11-11T20:52:19,718 DEBUG [RS:1;51ca66f7ee3c:42089 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8 2024-11-11T20:52:19,718 DEBUG [RS:0;51ca66f7ee3c:42859 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8 2024-11-11T20:52:19,718 DEBUG [RS:1;51ca66f7ee3c:42089 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-11T20:52:19,718 DEBUG [RS:0;51ca66f7ee3c:42859 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-11T20:52:19,718 DEBUG [RS:1;51ca66f7ee3c:42089 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T20:52:19,718 DEBUG [RS:0;51ca66f7ee3c:42859 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T20:52:19,719 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:19,721 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-11T20:52:19,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T20:52:19,724 DEBUG [RS:1;51ca66f7ee3c:42089 {}] zookeeper.ZKUtil(111): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/51ca66f7ee3c,42089,1731358338732 2024-11-11T20:52:19,724 DEBUG [RS:0;51ca66f7ee3c:42859 {}] zookeeper.ZKUtil(111): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/51ca66f7ee3c,42859,1731358338612 2024-11-11T20:52:19,724 WARN [RS:1;51ca66f7ee3c:42089 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T20:52:19,724 WARN [RS:0;51ca66f7ee3c:42859 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T20:52:19,725 INFO [RS:0;51ca66f7ee3c:42859 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:52:19,725 INFO [RS:1;51ca66f7ee3c:42089 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:52:19,725 DEBUG [RS:1;51ca66f7ee3c:42089 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,42089,1731358338732 2024-11-11T20:52:19,725 DEBUG [RS:0;51ca66f7ee3c:42859 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,42859,1731358338612 2024-11-11T20:52:19,725 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T20:52:19,725 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:19,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:19,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-11T20:52:19,729 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [51ca66f7ee3c,42859,1731358338612] 2024-11-11T20:52:19,729 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [51ca66f7ee3c,42089,1731358338732] 2024-11-11T20:52:19,731 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T20:52:19,731 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:19,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:19,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-11T20:52:19,743 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T20:52:19,743 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:19,743 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:19,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-11T20:52:19,749 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T20:52:19,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:19,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:19,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T20:52:19,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740 2024-11-11T20:52:19,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740 2024-11-11T20:52:19,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T20:52:19,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T20:52:19,752 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T20:52:19,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T20:52:19,760 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:52:19,760 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72116539, jitterRate=0.07462017238140106}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T20:52:19,765 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731358339719Initializing all the Stores at 1731358339719Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358339719Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358339721 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358339721Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358339721Cleaning up temporary data from old regions at 1731358339751 (+30 ms)Region opened successfully at 1731358339765 (+14 ms) 2024-11-11T20:52:19,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T20:52:19,766 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T20:52:19,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T20:52:19,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T20:52:19,766 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T20:52:19,766 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T20:52:19,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T20:52:19,768 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T20:52:19,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731358339766Disabling compacts and flushes for region at 1731358339766Disabling writes for close at 1731358339766Writing region close event to WAL at 1731358339767 (+1 ms)Closed at 1731358339768 (+1 ms) 2024-11-11T20:52:19,770 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T20:52:19,770 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T20:52:19,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T20:52:19,785 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T20:52:19,785 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T20:52:19,788 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:52:19,791 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T20:52:19,792 INFO [RS:1;51ca66f7ee3c:42089 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T20:52:19,792 INFO [RS:0;51ca66f7ee3c:42859 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T20:52:19,792 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,792 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,794 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T20:52:19,797 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T20:52:19,798 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T20:52:19,798 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T20:52:19,800 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,800 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,800 DEBUG [RS:0;51ca66f7ee3c:42859 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,800 DEBUG [RS:1;51ca66f7ee3c:42089 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,800 DEBUG [RS:0;51ca66f7ee3c:42859 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,800 DEBUG [RS:1;51ca66f7ee3c:42089 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,800 DEBUG [RS:0;51ca66f7ee3c:42859 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,800 DEBUG [RS:1;51ca66f7ee3c:42089 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,801 DEBUG [RS:0;51ca66f7ee3c:42859 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,801 DEBUG [RS:1;51ca66f7ee3c:42089 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,801 DEBUG [RS:0;51ca66f7ee3c:42859 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,801 DEBUG [RS:1;51ca66f7ee3c:42089 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,801 DEBUG [RS:0;51ca66f7ee3c:42859 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=2, maxPoolSize=2 2024-11-11T20:52:19,801 DEBUG [RS:0;51ca66f7ee3c:42859 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,801 DEBUG [RS:1;51ca66f7ee3c:42089 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=2, maxPoolSize=2 2024-11-11T20:52:19,801 DEBUG [RS:0;51ca66f7ee3c:42859 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,801 DEBUG [RS:1;51ca66f7ee3c:42089 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,801 DEBUG [RS:0;51ca66f7ee3c:42859 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,801 DEBUG [RS:1;51ca66f7ee3c:42089 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,801 DEBUG [RS:0;51ca66f7ee3c:42859 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,802 DEBUG [RS:0;51ca66f7ee3c:42859 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,802 DEBUG [RS:1;51ca66f7ee3c:42089 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,802 DEBUG [RS:0;51ca66f7ee3c:42859 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,802 DEBUG [RS:1;51ca66f7ee3c:42089 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,802 DEBUG [RS:0;51ca66f7ee3c:42859 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:52:19,802 DEBUG [RS:1;51ca66f7ee3c:42089 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,802 DEBUG [RS:0;51ca66f7ee3c:42859 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:52:19,802 DEBUG [RS:1;51ca66f7ee3c:42089 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:19,802 DEBUG [RS:1;51ca66f7ee3c:42089 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:52:19,803 DEBUG [RS:1;51ca66f7ee3c:42089 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:52:19,809 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,809 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,809 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,809 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,809 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,809 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,809 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,809 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,809 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,810 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,810 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,42859,1731358338612-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T20:52:19,810 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,42089,1731358338732-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T20:52:19,828 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T20:52:19,829 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,42859,1731358338612-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,830 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,830 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.Replication(171): 51ca66f7ee3c,42859,1731358338612 started 2024-11-11T20:52:19,831 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T20:52:19,831 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,42089,1731358338732-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,831 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,831 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.Replication(171): 51ca66f7ee3c,42089,1731358338732 started 2024-11-11T20:52:19,847 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,847 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.HRegionServer(1482): Serving as 51ca66f7ee3c,42859,1731358338612, RpcServer on 51ca66f7ee3c/172.17.0.2:42859, sessionid=0x100308f05e40001 2024-11-11T20:52:19,848 DEBUG [RS:0;51ca66f7ee3c:42859 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T20:52:19,848 DEBUG [RS:0;51ca66f7ee3c:42859 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 51ca66f7ee3c,42859,1731358338612 2024-11-11T20:52:19,848 DEBUG [RS:0;51ca66f7ee3c:42859 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,42859,1731358338612' 2024-11-11T20:52:19,848 DEBUG [RS:0;51ca66f7ee3c:42859 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T20:52:19,849 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:19,850 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.HRegionServer(1482): Serving as 51ca66f7ee3c,42089,1731358338732, RpcServer on 51ca66f7ee3c/172.17.0.2:42089, sessionid=0x100308f05e40002 2024-11-11T20:52:19,850 DEBUG [RS:0;51ca66f7ee3c:42859 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T20:52:19,850 DEBUG [RS:1;51ca66f7ee3c:42089 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T20:52:19,850 DEBUG [RS:1;51ca66f7ee3c:42089 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 51ca66f7ee3c,42089,1731358338732 2024-11-11T20:52:19,850 DEBUG [RS:1;51ca66f7ee3c:42089 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,42089,1731358338732' 2024-11-11T20:52:19,850 DEBUG [RS:1;51ca66f7ee3c:42089 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T20:52:19,850 DEBUG [RS:0;51ca66f7ee3c:42859 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T20:52:19,850 DEBUG [RS:0;51ca66f7ee3c:42859 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T20:52:19,851 DEBUG [RS:0;51ca66f7ee3c:42859 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 51ca66f7ee3c,42859,1731358338612 2024-11-11T20:52:19,851 DEBUG [RS:1;51ca66f7ee3c:42089 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T20:52:19,851 DEBUG [RS:0;51ca66f7ee3c:42859 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,42859,1731358338612' 2024-11-11T20:52:19,851 DEBUG [RS:0;51ca66f7ee3c:42859 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T20:52:19,851 DEBUG [RS:1;51ca66f7ee3c:42089 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T20:52:19,851 DEBUG [RS:1;51ca66f7ee3c:42089 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T20:52:19,851 DEBUG [RS:1;51ca66f7ee3c:42089 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 51ca66f7ee3c,42089,1731358338732 2024-11-11T20:52:19,851 DEBUG [RS:0;51ca66f7ee3c:42859 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T20:52:19,851 DEBUG [RS:1;51ca66f7ee3c:42089 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,42089,1731358338732' 2024-11-11T20:52:19,851 DEBUG [RS:1;51ca66f7ee3c:42089 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T20:52:19,852 DEBUG [RS:0;51ca66f7ee3c:42859 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T20:52:19,852 DEBUG [RS:1;51ca66f7ee3c:42089 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T20:52:19,852 INFO [RS:0;51ca66f7ee3c:42859 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T20:52:19,852 INFO [RS:0;51ca66f7ee3c:42859 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T20:52:19,852 DEBUG [RS:1;51ca66f7ee3c:42089 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T20:52:19,853 INFO [RS:1;51ca66f7ee3c:42089 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T20:52:19,853 INFO [RS:1;51ca66f7ee3c:42089 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T20:52:19,942 WARN [51ca66f7ee3c:39535 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T20:52:19,960 INFO [RS:1;51ca66f7ee3c:42089 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T20:52:19,960 INFO [RS:0;51ca66f7ee3c:42859 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T20:52:19,961 INFO [RS:1;51ca66f7ee3c:42089 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=51ca66f7ee3c%2C42089%2C1731358338732, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,42089,1731358338732, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/oldWALs, maxLogs=55 2024-11-11T20:52:19,961 INFO [RS:0;51ca66f7ee3c:42859 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=51ca66f7ee3c%2C42859%2C1731358338612, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,42859,1731358338612, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/oldWALs, maxLogs=55 2024-11-11T20:52:19,987 INFO [RS:1;51ca66f7ee3c:42089 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,42089,1731358338732/51ca66f7ee3c%2C42089%2C1731358338732.1731358339966 2024-11-11T20:52:19,987 INFO [RS:0;51ca66f7ee3c:42859 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,42859,1731358338612/51ca66f7ee3c%2C42859%2C1731358338612.1731358339966 2024-11-11T20:52:19,988 DEBUG [RS:1;51ca66f7ee3c:42089 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T20:52:19,988 DEBUG [RS:0;51ca66f7ee3c:42859 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T20:52:20,197 DEBUG [51ca66f7ee3c:39535 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-11T20:52:20,212 DEBUG [51ca66f7ee3c:39535 {}] balancer.BalancerClusterState(204): Hosts are {51ca66f7ee3c=0} racks are {/default-rack=0} 2024-11-11T20:52:20,215 DEBUG [51ca66f7ee3c:39535 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T20:52:20,215 DEBUG [51ca66f7ee3c:39535 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T20:52:20,215 DEBUG [51ca66f7ee3c:39535 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T20:52:20,216 DEBUG [51ca66f7ee3c:39535 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T20:52:20,216 INFO [51ca66f7ee3c:39535 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T20:52:20,216 INFO [51ca66f7ee3c:39535 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T20:52:20,216 DEBUG [51ca66f7ee3c:39535 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T20:52:20,223 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=51ca66f7ee3c,42089,1731358338732 2024-11-11T20:52:20,228 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,42089,1731358338732, state=OPENING 2024-11-11T20:52:20,233 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T20:52:20,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:20,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:20,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:20,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:20,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:20,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:20,237 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:52:20,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,42089,1731358338732}] 2024-11-11T20:52:20,410 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T20:52:20,414 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39237, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T20:52:20,423 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T20:52:20,423 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:52:20,424 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T20:52:20,426 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=51ca66f7ee3c%2C42089%2C1731358338732.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,42089,1731358338732, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/oldWALs, maxLogs=55 2024-11-11T20:52:20,451 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,42089,1731358338732/51ca66f7ee3c%2C42089%2C1731358338732.meta.1731358340429.meta 2024-11-11T20:52:20,452 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T20:52:20,452 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:52:20,454 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T20:52:20,456 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T20:52:20,460 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T20:52:20,463 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T20:52:20,464 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:20,464 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T20:52:20,464 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T20:52:20,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-11T20:52:20,467 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T20:52:20,467 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:20,467 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:20,467 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-11T20:52:20,468 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T20:52:20,468 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:20,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:20,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-11T20:52:20,469 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T20:52:20,469 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:20,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:20,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-11T20:52:20,470 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T20:52:20,470 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:20,470 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:20,471 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T20:52:20,472 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740 2024-11-11T20:52:20,472 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740 2024-11-11T20:52:20,472 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T20:52:20,473 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T20:52:20,473 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T20:52:20,474 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T20:52:20,474 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73475019, jitterRate=0.09486310184001923}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T20:52:20,475 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T20:52:20,476 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731358340465Writing region info on filesystem at 1731358340465Initializing all the Stores at 1731358340465Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358340465Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358340466 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358340466Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358340466Cleaning up temporary data from old regions at 1731358340473 (+7 ms)Running coprocessor post-open hooks at 1731358340475 (+2 ms)Region opened successfully at 1731358340476 (+1 ms) 2024-11-11T20:52:20,481 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731358340401 2024-11-11T20:52:20,491 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T20:52:20,492 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T20:52:20,493 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,42089,1731358338732 2024-11-11T20:52:20,494 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,42089,1731358338732, state=OPEN 2024-11-11T20:52:20,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:52:20,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:52:20,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:52:20,496 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:20,496 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:20,496 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:20,496 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,42089,1731358338732 2024-11-11T20:52:20,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T20:52:20,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,42089,1731358338732 in 257 msec 2024-11-11T20:52:20,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T20:52:20,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 723 msec 2024-11-11T20:52:20,505 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T20:52:20,505 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T20:52:20,523 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:20,524 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,42089,1731358338732, seqNum=-1] 2024-11-11T20:52:20,542 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:20,544 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59661, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:20,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 988 msec 2024-11-11T20:52:20,560 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731358340560, completionTime=-1 2024-11-11T20:52:20,563 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-11T20:52:20,563 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:52:20,587 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:52:20,587 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358400587 2024-11-11T20:52:20,588 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358460587 2024-11-11T20:52:20,588 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-11T20:52:20,590 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:52:20,596 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,39535,1731358337804-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:20,596 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,39535,1731358337804-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:20,596 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,39535,1731358337804-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:20,597 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-51ca66f7ee3c:39535, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:20,598 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:20,598 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:20,604 DEBUG [master/51ca66f7ee3c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T20:52:20,626 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.798sec 2024-11-11T20:52:20,628 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T20:52:20,629 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T20:52:20,630 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T20:52:20,631 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T20:52:20,631 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T20:52:20,632 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,39535,1731358337804-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T20:52:20,632 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,39535,1731358337804-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T20:52:20,633 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T20:52:20,634 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T20:52:20,635 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,39535,1731358337804-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:20,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:20,678 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T20:52:20,678 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T20:52:20,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,39535,-1 for getting cluster id 2024-11-11T20:52:20,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:52:20,693 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6c67f0b4-4791-4b2c-a567-2be657e06ec3' 2024-11-11T20:52:20,696 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:52:20,696 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6c67f0b4-4791-4b2c-a567-2be657e06ec3" 2024-11-11T20:52:20,697 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ba301dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:20,697 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,39535,-1] 2024-11-11T20:52:20,699 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:52:20,701 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:20,703 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:52:20,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:20,706 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:20,713 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,42089,1731358338732, seqNum=-1] 2024-11-11T20:52:20,714 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:20,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:20,744 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=51ca66f7ee3c,39535,1731358337804 2024-11-11T20:52:20,748 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:52:20,755 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,39535,1731358337804 2024-11-11T20:52:20,757 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1b610038 2024-11-11T20:52:20,758 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:52:20,761 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:52:20,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731358340744', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:52:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731358340744 2024-11-11T20:52:20,773 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731358340744 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T20:52:20,775 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731358340744" procId is: 4 2024-11-11T20:52:20,775 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:20,776 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731358340744 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T20:52:20,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:52:20,787 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731358340744-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e220ae0708f4f613444d3a7751d2203f, NAME => 'testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731358340744', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8 2024-11-11T20:52:20,793 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731358340744-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:20,793 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731358340744-pool-0 {}] regionserver.HRegion(1722): Closing e220ae0708f4f613444d3a7751d2203f, disabling compactions & flushes 2024-11-11T20:52:20,793 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731358340744-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:20,793 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731358340744-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:20,793 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731358340744-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. after waiting 0 ms 2024-11-11T20:52:20,793 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731358340744-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:20,793 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731358340744-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:20,793 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731358340744-pool-0 {}] regionserver.HRegion(1676): Region close journal for e220ae0708f4f613444d3a7751d2203f: Waiting for close lock at 1731358340793Disabling compacts and flushes for region at 1731358340793Disabling writes for close at 1731358340793Writing region close event to WAL at 1731358340793Closed at 1731358340793 2024-11-11T20:52:20,795 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731358340744 execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T20:52:20,800 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731358340795"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358340795"}]},"ts":"1731358340795"} 2024-11-11T20:52:20,804 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T20:52:20,806 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731358340744 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T20:52:20,808 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731358340744","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358340806"}]},"ts":"1731358340806"} 2024-11-11T20:52:20,813 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731358340744, state=ENABLING in hbase:meta 2024-11-11T20:52:20,814 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {51ca66f7ee3c=0} racks are {/default-rack=0} 2024-11-11T20:52:20,814 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T20:52:20,814 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T20:52:20,814 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T20:52:20,814 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T20:52:20,814 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T20:52:20,815 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T20:52:20,815 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T20:52:20,816 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731358340744, region=e220ae0708f4f613444d3a7751d2203f, ASSIGN}] 2024-11-11T20:52:20,818 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731358340744, region=e220ae0708f4f613444d3a7751d2203f, ASSIGN 2024-11-11T20:52:20,819 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731358340744, region=e220ae0708f4f613444d3a7751d2203f, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,42089,1731358338732; forceNewPlan=false, retain=false 2024-11-11T20:52:20,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:52:20,971 INFO [51ca66f7ee3c:39535 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-11T20:52:20,972 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e220ae0708f4f613444d3a7751d2203f, regionState=OPENING, regionLocation=51ca66f7ee3c,42089,1731358338732 2024-11-11T20:52:20,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731358340744, region=e220ae0708f4f613444d3a7751d2203f, ASSIGN because future has completed 2024-11-11T20:52:20,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e220ae0708f4f613444d3a7751d2203f, server=51ca66f7ee3c,42089,1731358338732}] 2024-11-11T20:52:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:52:21,140 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:21,141 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e220ae0708f4f613444d3a7751d2203f, NAME => 'testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:52:21,143 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731358340744 e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:21,144 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:21,144 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:21,144 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:21,146 INFO [StoreOpener-e220ae0708f4f613444d3a7751d2203f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:21,150 INFO [StoreOpener-e220ae0708f4f613444d3a7751d2203f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e220ae0708f4f613444d3a7751d2203f columnFamilyName family 2024-11-11T20:52:21,150 DEBUG [StoreOpener-e220ae0708f4f613444d3a7751d2203f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:21,150 INFO [StoreOpener-e220ae0708f4f613444d3a7751d2203f-1 {}] regionserver.HStore(327): Store=e220ae0708f4f613444d3a7751d2203f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:21,150 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:21,152 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:21,152 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:21,152 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:21,152 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:21,153 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:21,159 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:52:21,160 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e220ae0708f4f613444d3a7751d2203f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59717396, jitterRate=-0.11014145612716675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:52:21,160 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:21,161 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e220ae0708f4f613444d3a7751d2203f: Running coprocessor pre-open hook at 1731358341145Writing region info on filesystem at 1731358341145Initializing all the Stores at 1731358341145Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358341146 (+1 ms)Cleaning up temporary data from old regions at 1731358341152 (+6 ms)Running coprocessor post-open hooks at 1731358341160 (+8 ms)Region opened successfully at 1731358341161 (+1 ms) 2024-11-11T20:52:21,162 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f., pid=6, masterSystemTime=1731358341133 2024-11-11T20:52:21,166 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:21,166 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:21,166 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e220ae0708f4f613444d3a7751d2203f, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,42089,1731358338732 2024-11-11T20:52:21,169 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e220ae0708f4f613444d3a7751d2203f, server=51ca66f7ee3c,42089,1731358338732 because future has completed 2024-11-11T20:52:21,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T20:52:21,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e220ae0708f4f613444d3a7751d2203f, server=51ca66f7ee3c,42089,1731358338732 in 193 msec 2024-11-11T20:52:21,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T20:52:21,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731358340744, region=e220ae0708f4f613444d3a7751d2203f, ASSIGN in 357 msec 2024-11-11T20:52:21,176 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731358340744 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T20:52:21,177 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731358340744","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358341177"}]},"ts":"1731358341177"} 2024-11-11T20:52:21,179 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731358340744, state=ENABLED in hbase:meta 2024-11-11T20:52:21,180 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731358340744 execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T20:52:21,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731358340744 in 412 msec 2024-11-11T20:52:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:52:21,412 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731358340744 completed 2024-11-11T20:52:21,412 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731358340744 get assigned. Timeout = 60000ms 2024-11-11T20:52:21,416 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T20:52:21,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731358340744 assigned to meta. Checking AM states. 2024-11-11T20:52:21,426 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T20:52:21,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731358340744 assigned. 2024-11-11T20:52:21,438 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731358340744', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f., hostname=51ca66f7ee3c,42089,1731358338732, seqNum=2] 2024-11-11T20:52:21,449 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e220ae0708f4f613444d3a7751d2203f 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-11T20:52:21,472 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bdff2266b0814b2ebd84a50cd6f3fd05 2024-11-11T20:52:21,507 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/.tmp/family/bdff2266b0814b2ebd84a50cd6f3fd05 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-11T20:52:21,511 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/.tmp/family/bdff2266b0814b2ebd84a50cd6f3fd05 2024-11-11T20:52:21,540 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/.tmp/family/bdff2266b0814b2ebd84a50cd6f3fd05 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/family/bdff2266b0814b2ebd84a50cd6f3fd05 2024-11-11T20:52:21,542 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/family/bdff2266b0814b2ebd84a50cd6f3fd05, delay=966 ms 2024-11-11T20:52:21,543 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/family/bdff2266b0814b2ebd84a50cd6f3fd05, entries=2, sequenceid=6, filesize=4.9 K 2024-11-11T20:52:21,549 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for e220ae0708f4f613444d3a7751d2203f in 100ms, sequenceid=6, compaction requested=false 2024-11-11T20:52:21,550 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731358340744' 2024-11-11T20:52:21,551 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e220ae0708f4f613444d3a7751d2203f: 2024-11-11T20:52:22,511 DEBUG [hfile-prefetch-1731358341542 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key bdff2266b0814b2ebd84a50cd6f3fd05_0. 2024-11-11T20:52:22,516 DEBUG [hfile-prefetch-1731358341542 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key bdff2266b0814b2ebd84a50cd6f3fd05_117. 2024-11-11T20:52:22,518 DEBUG [hfile-prefetch-1731358341542 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/family/bdff2266b0814b2ebd84a50cd6f3fd05, with total blocks 2, and data blocks 1 2024-11-11T20:52:22,518 DEBUG [hfile-prefetch-1731358341542 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-11T20:52:22,518 DEBUG [hfile-prefetch-1731358341542 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/family/bdff2266b0814b2ebd84a50cd6f3fd05, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-11T20:52:22,518 DEBUG [hfile-prefetch-1731358341542 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/family/bdff2266b0814b2ebd84a50cd6f3fd05. Will sleep 100ms and try the verification again. 2024-11-11T20:52:22,551 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-11T20:52:22,551 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-11T20:52:22,552 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T20:52:22,552 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T20:52:22,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=e220ae0708f4f613444d3a7751d2203f, source=51ca66f7ee3c,42089,1731358338732, destination=51ca66f7ee3c,42859,1731358338612, warming up region on 51ca66f7ee3c,42859,1731358338612 2024-11-11T20:52:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T20:52:22,567 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=e220ae0708f4f613444d3a7751d2203f, source=51ca66f7ee3c,42089,1731358338732, destination=51ca66f7ee3c,42859,1731358338612, running balancer 2024-11-11T20:52:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731358340744, region=e220ae0708f4f613444d3a7751d2203f, REOPEN/MOVE 2024-11-11T20:52:22,569 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731358340744, region=e220ae0708f4f613444d3a7751d2203f, REOPEN/MOVE 2024-11-11T20:52:22,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32907, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T20:52:22,572 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e220ae0708f4f613444d3a7751d2203f, regionState=CLOSING, regionLocation=51ca66f7ee3c,42089,1731358338732 2024-11-11T20:52:22,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42859 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42859 {}] regionserver.HRegion(7855): Warmup {ENCODED => e220ae0708f4f613444d3a7751d2203f, NAME => 'testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:52:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42859 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:22,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731358340744, region=e220ae0708f4f613444d3a7751d2203f, REOPEN/MOVE because future has completed 2024-11-11T20:52:22,579 INFO [StoreOpener-e220ae0708f4f613444d3a7751d2203f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:22,580 INFO [StoreOpener-e220ae0708f4f613444d3a7751d2203f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e220ae0708f4f613444d3a7751d2203f columnFamilyName family 2024-11-11T20:52:22,580 DEBUG [StoreOpener-e220ae0708f4f613444d3a7751d2203f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:22,581 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-11T20:52:22,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure e220ae0708f4f613444d3a7751d2203f, server=51ca66f7ee3c,42089,1731358338732}] 2024-11-11T20:52:22,587 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-11T20:52:22,587 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731358342551 2024-11-11T20:52:22,587 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-11T20:52:22,589 DEBUG [StoreOpener-e220ae0708f4f613444d3a7751d2203f-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/family/bdff2266b0814b2ebd84a50cd6f3fd05 2024-11-11T20:52:22,589 INFO [StoreOpener-e220ae0708f4f613444d3a7751d2203f-1 {}] regionserver.HStore(327): Store=e220ae0708f4f613444d3a7751d2203f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42859 {}] regionserver.HRegion(1722): Closing e220ae0708f4f613444d3a7751d2203f, disabling compactions & flushes 2024-11-11T20:52:22,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42859 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42859 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42859 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. after waiting 0 ms 2024-11-11T20:52:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42859 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:22,591 DEBUG [StoreFileCloser-e220ae0708f4f613444d3a7751d2203f-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/family/bdff2266b0814b2ebd84a50cd6f3fd05 2024-11-11T20:52:22,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42859 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42859 {}] regionserver.HRegion(1676): Region close journal for e220ae0708f4f613444d3a7751d2203f: Waiting for close lock at 1731358342590Disabling compacts and flushes for region at 1731358342590Disabling writes for close at 1731358342590Writing region close event to WAL at 1731358342592 (+2 ms)Closed at 1731358342592 2024-11-11T20:52:22,594 WARN [hfile-prefetch-1731358341542 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/family/bdff2266b0814b2ebd84a50cd6f3fd05, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-11T20:52:22,595 DEBUG [hfile-prefetch-1731358341542 {}] hfile.PrefetchExecutor(121): Prefetch completed for bdff2266b0814b2ebd84a50cd6f3fd05 2024-11-11T20:52:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39535 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-11T20:52:22,741 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:22,741 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-11T20:52:22,741 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing e220ae0708f4f613444d3a7751d2203f, disabling compactions & flushes 2024-11-11T20:52:22,741 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:22,742 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:22,742 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. after waiting 0 ms 2024-11-11T20:52:22,742 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:22,743 DEBUG [StoreFileCloser-e220ae0708f4f613444d3a7751d2203f-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file bdff2266b0814b2ebd84a50cd6f3fd05, starting offset: 0, end offset: 9223372036854775807 2024-11-11T20:52:22,744 DEBUG [StoreFileCloser-e220ae0708f4f613444d3a7751d2203f-family-1 {}] bucket.BucketCache(857): removed key bdff2266b0814b2ebd84a50cd6f3fd05_0 from back map with offset lock 2094080 in the evict process 2024-11-11T20:52:22,749 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-11T20:52:22,750 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:22,750 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for e220ae0708f4f613444d3a7751d2203f: Waiting for close lock at 1731358342741Running coprocessor pre-close hooks at 1731358342741Disabling compacts and flushes for region at 1731358342741Disabling writes for close at 1731358342742 (+1 ms)Writing region close event to WAL at 1731358342745 (+3 ms)Running coprocessor post-close hooks at 1731358342749 (+4 ms)Closed at 1731358342750 (+1 ms) 2024-11-11T20:52:22,750 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding e220ae0708f4f613444d3a7751d2203f move to 51ca66f7ee3c,42859,1731358338612 record at close sequenceid=6 2024-11-11T20:52:22,754 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:22,754 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e220ae0708f4f613444d3a7751d2203f, regionState=CLOSED 2024-11-11T20:52:22,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure e220ae0708f4f613444d3a7751d2203f, server=51ca66f7ee3c,42089,1731358338732 because future has completed 2024-11-11T20:52:22,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T20:52:22,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure e220ae0708f4f613444d3a7751d2203f, server=51ca66f7ee3c,42089,1731358338732 in 177 msec 2024-11-11T20:52:22,761 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731358340744, region=e220ae0708f4f613444d3a7751d2203f, REOPEN/MOVE; state=CLOSED, location=51ca66f7ee3c,42859,1731358338612; forceNewPlan=false, retain=false 2024-11-11T20:52:22,788 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-11T20:52:22,788 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-11T20:52:22,788 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T20:52:22,788 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T20:52:22,788 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-11T20:52:22,789 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-11T20:52:22,789 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T20:52:22,789 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T20:52:22,791 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-11T20:52:22,791 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731358342788 2024-11-11T20:52:22,791 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-11T20:52:22,790 WARN [Thread-92 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-11T20:52:22,790 ERROR [bucket-cache-persister {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1679) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1497) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2024-11-11T20:52:22,791 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-11T20:52:22,791 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731358342789 2024-11-11T20:52:22,791 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-11T20:52:22,912 INFO [51ca66f7ee3c:39535 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-11T20:52:22,913 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e220ae0708f4f613444d3a7751d2203f, regionState=OPENING, regionLocation=51ca66f7ee3c,42859,1731358338612 2024-11-11T20:52:22,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731358340744, region=e220ae0708f4f613444d3a7751d2203f, REOPEN/MOVE because future has completed 2024-11-11T20:52:22,925 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e220ae0708f4f613444d3a7751d2203f, server=51ca66f7ee3c,42859,1731358338612}] 2024-11-11T20:52:23,082 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:23,082 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => e220ae0708f4f613444d3a7751d2203f, NAME => 'testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:52:23,084 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731358340744 e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:23,084 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:23,084 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:23,085 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:23,087 INFO [StoreOpener-e220ae0708f4f613444d3a7751d2203f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:23,089 INFO [StoreOpener-e220ae0708f4f613444d3a7751d2203f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e220ae0708f4f613444d3a7751d2203f columnFamilyName family 2024-11-11T20:52:23,090 DEBUG [StoreOpener-e220ae0708f4f613444d3a7751d2203f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:23,093 DEBUG [StoreFileOpener-e220ae0708f4f613444d3a7751d2203f-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/family/bdff2266b0814b2ebd84a50cd6f3fd05, delay=925 ms 2024-11-11T20:52:23,094 DEBUG [StoreOpener-e220ae0708f4f613444d3a7751d2203f-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/family/bdff2266b0814b2ebd84a50cd6f3fd05 2024-11-11T20:52:23,094 INFO [StoreOpener-e220ae0708f4f613444d3a7751d2203f-1 {}] regionserver.HStore(327): Store=e220ae0708f4f613444d3a7751d2203f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:23,094 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:23,095 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:23,096 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:23,096 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:23,096 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:23,099 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:23,100 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened e220ae0708f4f613444d3a7751d2203f; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65055853, jitterRate=-0.03059224784374237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:52:23,100 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:23,101 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for e220ae0708f4f613444d3a7751d2203f: Running coprocessor pre-open hook at 1731358343085Writing region info on filesystem at 1731358343085Initializing all the Stores at 1731358343086 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358343086Cleaning up temporary data from old regions at 1731358343096 (+10 ms)Running coprocessor post-open hooks at 1731358343100 (+4 ms)Region opened successfully at 1731358343101 (+1 ms) 2024-11-11T20:52:23,102 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f., pid=9, masterSystemTime=1731358343077 2024-11-11T20:52:23,104 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:23,105 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:23,105 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e220ae0708f4f613444d3a7751d2203f, regionState=OPEN, openSeqNum=10, regionLocation=51ca66f7ee3c,42859,1731358338612 2024-11-11T20:52:23,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e220ae0708f4f613444d3a7751d2203f, server=51ca66f7ee3c,42859,1731358338612 because future has completed 2024-11-11T20:52:23,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-11T20:52:23,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure e220ae0708f4f613444d3a7751d2203f, server=51ca66f7ee3c,42859,1731358338612 in 185 msec 2024-11-11T20:52:23,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731358340744, region=e220ae0708f4f613444d3a7751d2203f, REOPEN/MOVE in 544 msec 2024-11-11T20:52:23,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T20:52:23,122 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:52:23,123 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:52:23,126 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:23,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:23,127 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:52:23,127 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T20:52:23,127 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1482236467, stopped=false 2024-11-11T20:52:23,128 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=51ca66f7ee3c,39535,1731358337804 2024-11-11T20:52:23,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T20:52:23,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T20:52:23,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T20:52:23,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,129 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:52:23,130 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:52:23,130 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:52:23,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:23,130 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:52:23,131 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:52:23,131 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.HRegionServer(878): Closing user regions 2024-11-11T20:52:23,131 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.HRegionServer(3091): Received CLOSE for e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:23,131 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '51ca66f7ee3c,42859,1731358338612' ***** 2024-11-11T20:52:23,131 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T20:52:23,131 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '51ca66f7ee3c,42089,1731358338732' ***** 2024-11-11T20:52:23,131 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T20:52:23,131 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:52:23,131 INFO [RS:1;51ca66f7ee3c:42089 {}] bucket.BucketCache(1739): Disabling cache 2024-11-11T20:52:23,132 INFO [RS:1;51ca66f7ee3c:42089 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/bucket.persistence 2024-11-11T20:52:23,132 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-11T20:52:23,132 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-11T20:52:23,132 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-11T20:52:23,132 INFO [RS:1;51ca66f7ee3c:42089 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-11T20:52:23,132 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-11T20:52:23,133 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-11T20:52:23,133 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e220ae0708f4f613444d3a7751d2203f, disabling compactions & flushes 2024-11-11T20:52:23,133 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:23,133 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:23,133 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. after waiting 0 ms 2024-11-11T20:52:23,133 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:23,134 DEBUG [StoreFileCloser-e220ae0708f4f613444d3a7751d2203f-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/family/bdff2266b0814b2ebd84a50cd6f3fd05 2024-11-11T20:52:23,135 INFO [RS:0;51ca66f7ee3c:42859 {}] bucket.BucketCache(1739): Disabling cache 2024-11-11T20:52:23,135 INFO [RS:0;51ca66f7ee3c:42859 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/bucket.persistence 2024-11-11T20:52:23,135 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-11T20:52:23,135 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-11T20:52:23,135 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-11T20:52:23,135 INFO [RS:0;51ca66f7ee3c:42859 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-11T20:52:23,135 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-11T20:52:23,139 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnRegionMove1731358340744/e220ae0708f4f613444d3a7751d2203f/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-11T20:52:23,140 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:23,140 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e220ae0708f4f613444d3a7751d2203f: Waiting for close lock at 1731358343132Running coprocessor pre-close hooks at 1731358343133 (+1 ms)Disabling compacts and flushes for region at 1731358343133Disabling writes for close at 1731358343133Writing region close event to WAL at 1731358343135 (+2 ms)Running coprocessor post-close hooks at 1731358343140 (+5 ms)Closed at 1731358343140 2024-11-11T20:52:23,141 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39535 {}] assignment.AssignmentManager(1314): RegionServer CLOSED e220ae0708f4f613444d3a7751d2203f 2024-11-11T20:52:23,142 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f. 2024-11-11T20:52:23,143 DEBUG [RS:1;51ca66f7ee3c:42089 {}] bucket.BucketCache(1488): Thread RS:1;51ca66f7ee3c:42089 started persisting bucket cache to file 2024-11-11T20:52:23,143 DEBUG [RS:1;51ca66f7ee3c:42089 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T20:52:23,143 DEBUG [RS:1;51ca66f7ee3c:42089 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T20:52:23,145 DEBUG [RS:1;51ca66f7ee3c:42089 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-11T20:52:23,145 DEBUG [RS:0;51ca66f7ee3c:42859 {}] bucket.BucketCache(1488): Thread RS:0;51ca66f7ee3c:42859 started persisting bucket cache to file 2024-11-11T20:52:23,145 DEBUG [RS:1;51ca66f7ee3c:42089 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731358343143 2024-11-11T20:52:23,145 DEBUG [RS:1;51ca66f7ee3c:42089 {}] bucket.BucketCache(1510): Thread RS:1;51ca66f7ee3c:42089 finished persisting bucket cache to file, renaming 2024-11-11T20:52:23,145 DEBUG [RS:0;51ca66f7ee3c:42859 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T20:52:23,145 DEBUG [RS:0;51ca66f7ee3c:42859 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T20:52:23,146 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T20:52:23,146 INFO [RS:1;51ca66f7ee3c:42089 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T20:52:23,146 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T20:52:23,146 INFO [RS:1;51ca66f7ee3c:42089 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T20:52:23,146 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.HRegionServer(959): stopping server 51ca66f7ee3c,42089,1731358338732 2024-11-11T20:52:23,146 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T20:52:23,146 INFO [RS:1;51ca66f7ee3c:42089 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;51ca66f7ee3c:42089. 2024-11-11T20:52:23,146 DEBUG [RS:1;51ca66f7ee3c:42089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:52:23,147 DEBUG [RS:1;51ca66f7ee3c:42089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:23,147 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T20:52:23,147 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T20:52:23,147 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T20:52:23,147 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T20:52:23,147 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T20:52:23,147 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-11T20:52:23,148 DEBUG [RS:1;51ca66f7ee3c:42089 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-11T20:52:23,148 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T20:52:23,148 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T20:52:23,148 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T20:52:23,148 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T20:52:23,148 DEBUG [RS:1;51ca66f7ee3c:42089 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:52:23,148 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T20:52:23,148 DEBUG [RS:0;51ca66f7ee3c:42859 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-11T20:52:23,148 DEBUG [RS:0;51ca66f7ee3c:42859 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731358343145 2024-11-11T20:52:23,148 DEBUG [RS:0;51ca66f7ee3c:42859 {}] bucket.BucketCache(1510): Thread RS:0;51ca66f7ee3c:42859 finished persisting bucket cache to file, renaming 2024-11-11T20:52:23,148 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-11T20:52:23,148 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T20:52:23,149 INFO [RS:0;51ca66f7ee3c:42859 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T20:52:23,149 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T20:52:23,149 INFO [RS:0;51ca66f7ee3c:42859 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T20:52:23,149 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.HRegionServer(959): stopping server 51ca66f7ee3c,42859,1731358338612 2024-11-11T20:52:23,149 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T20:52:23,149 INFO [RS:0;51ca66f7ee3c:42859 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;51ca66f7ee3c:42859. 2024-11-11T20:52:23,149 DEBUG [RS:0;51ca66f7ee3c:42859 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:52:23,149 DEBUG [RS:0;51ca66f7ee3c:42859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:23,149 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.HRegionServer(976): stopping server 51ca66f7ee3c,42859,1731358338612; all regions closed. 2024-11-11T20:52:23,154 DEBUG [RS:0;51ca66f7ee3c:42859 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/oldWALs 2024-11-11T20:52:23,154 INFO [RS:0;51ca66f7ee3c:42859 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51ca66f7ee3c%2C42859%2C1731358338612:(num 1731358339966) 2024-11-11T20:52:23,154 DEBUG [RS:0;51ca66f7ee3c:42859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:23,154 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:52:23,155 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T20:52:23,155 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.ChoreService(370): Chore service for: regionserver/51ca66f7ee3c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T20:52:23,155 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T20:52:23,155 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T20:52:23,155 INFO [regionserver/51ca66f7ee3c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T20:52:23,155 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T20:52:23,155 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T20:52:23,156 INFO [RS:0;51ca66f7ee3c:42859 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42859 2024-11-11T20:52:23,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/51ca66f7ee3c,42859,1731358338612 2024-11-11T20:52:23,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T20:52:23,159 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T20:52:23,160 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9e87eaaac09f43b28c8877709c4cd740 2024-11-11T20:52:23,161 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [51ca66f7ee3c,42859,1731358338612] 2024-11-11T20:52:23,162 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/51ca66f7ee3c,42859,1731358338612 already deleted, retry=false 2024-11-11T20:52:23,163 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 51ca66f7ee3c,42859,1731358338612 expired; onlineServers=1 2024-11-11T20:52:23,177 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/info/9e87eaaac09f43b28c8877709c4cd740 is 205, key is testBlockEvictionOnRegionMove1731358340744,,1731358340762.e220ae0708f4f613444d3a7751d2203f./info:regioninfo/1731358343105/Put/seqid=0 2024-11-11T20:52:23,178 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/info/9e87eaaac09f43b28c8877709c4cd740 2024-11-11T20:52:23,187 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a580d79ce46c450cb0e71a0fae8ac88d 2024-11-11T20:52:23,199 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/ns/a580d79ce46c450cb0e71a0fae8ac88d is 43, key is default/ns:d/1731358340548/Put/seqid=0 2024-11-11T20:52:23,199 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/ns/a580d79ce46c450cb0e71a0fae8ac88d 2024-11-11T20:52:23,207 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2f0c9b7a3f464d04bde76209b369afa9 2024-11-11T20:52:23,214 INFO [regionserver/51ca66f7ee3c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:52:23,214 INFO [regionserver/51ca66f7ee3c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:52:23,219 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/table/2f0c9b7a3f464d04bde76209b369afa9 is 78, key is testBlockEvictionOnRegionMove1731358340744/table:state/1731358341177/Put/seqid=0 2024-11-11T20:52:23,220 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/table/2f0c9b7a3f464d04bde76209b369afa9 2024-11-11T20:52:23,223 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/info/9e87eaaac09f43b28c8877709c4cd740 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/info/9e87eaaac09f43b28c8877709c4cd740 2024-11-11T20:52:23,224 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/info/9e87eaaac09f43b28c8877709c4cd740, delay=982 ms 2024-11-11T20:52:23,225 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/info/9e87eaaac09f43b28c8877709c4cd740, entries=15, sequenceid=15, filesize=7.6 K 2024-11-11T20:52:23,225 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/ns/a580d79ce46c450cb0e71a0fae8ac88d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/ns/a580d79ce46c450cb0e71a0fae8ac88d 2024-11-11T20:52:23,226 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/ns/a580d79ce46c450cb0e71a0fae8ac88d, delay=913 ms 2024-11-11T20:52:23,228 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/ns/a580d79ce46c450cb0e71a0fae8ac88d, entries=2, sequenceid=15, filesize=5.0 K 2024-11-11T20:52:23,228 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/table/2f0c9b7a3f464d04bde76209b369afa9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/table/2f0c9b7a3f464d04bde76209b369afa9 2024-11-11T20:52:23,229 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/table/2f0c9b7a3f464d04bde76209b369afa9, delay=903 ms 2024-11-11T20:52:23,230 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/table/2f0c9b7a3f464d04bde76209b369afa9, entries=2, sequenceid=15, filesize=5.3 K 2024-11-11T20:52:23,231 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 83ms, sequenceid=15, compaction requested=false 2024-11-11T20:52:23,231 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T20:52:23,233 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/info/9e87eaaac09f43b28c8877709c4cd740 2024-11-11T20:52:23,234 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/ns/a580d79ce46c450cb0e71a0fae8ac88d 2024-11-11T20:52:23,239 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/table/2f0c9b7a3f464d04bde76209b369afa9 2024-11-11T20:52:23,243 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-11T20:52:23,243 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T20:52:23,244 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T20:52:23,244 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731358343147Running coprocessor pre-close hooks at 1731358343148 (+1 ms)Disabling compacts and flushes for region at 1731358343148Disabling writes for close at 1731358343148Obtaining lock to block concurrent updates at 1731358343148Preparing flush snapshotting stores in 1588230740 at 1731358343148Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731358343149 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731358343149Flushing 1588230740/info: creating writer at 1731358343150 (+1 ms)Flushing 1588230740/info: appending metadata at 1731358343176 (+26 ms)Flushing 1588230740/info: closing flushed file at 1731358343176Flushing 1588230740/ns: creating writer at 1731358343181 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731358343198 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731358343198Flushing 1588230740/table: creating writer at 1731358343202 (+4 ms)Flushing 1588230740/table: appending metadata at 1731358343219 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731358343219Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28d22300: reopening flushed file at 1731358343222 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b938a94: reopening flushed file at 1731358343225 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@233dc517: reopening flushed file at 1731358343228 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 83ms, sequenceid=15, compaction requested=false at 1731358343231 (+3 ms)Writing region close event to WAL at 1731358343239 (+8 ms)Running coprocessor post-close hooks at 1731358343243 (+4 ms)Closed at 1731358343244 (+1 ms) 2024-11-11T20:52:23,244 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T20:52:23,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:52:23,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42859-0x100308f05e40001, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:52:23,262 INFO [RS:0;51ca66f7ee3c:42859 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T20:52:23,262 INFO [RS:0;51ca66f7ee3c:42859 {}] regionserver.HRegionServer(1031): Exiting; stopping=51ca66f7ee3c,42859,1731358338612; zookeeper connection closed. 2024-11-11T20:52:23,262 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@32ca2105 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@32ca2105 2024-11-11T20:52:23,348 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.HRegionServer(976): stopping server 51ca66f7ee3c,42089,1731358338732; all regions closed. 2024-11-11T20:52:23,353 DEBUG [RS:1;51ca66f7ee3c:42089 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/oldWALs 2024-11-11T20:52:23,353 INFO [RS:1;51ca66f7ee3c:42089 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51ca66f7ee3c%2C42089%2C1731358338732.meta:.meta(num 1731358340429) 2024-11-11T20:52:23,356 DEBUG [RS:1;51ca66f7ee3c:42089 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/oldWALs 2024-11-11T20:52:23,357 INFO [RS:1;51ca66f7ee3c:42089 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51ca66f7ee3c%2C42089%2C1731358338732:(num 1731358339966) 2024-11-11T20:52:23,357 DEBUG [RS:1;51ca66f7ee3c:42089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:23,357 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:52:23,357 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T20:52:23,357 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.ChoreService(370): Chore service for: regionserver/51ca66f7ee3c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T20:52:23,358 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T20:52:23,358 INFO [regionserver/51ca66f7ee3c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T20:52:23,358 INFO [RS:1;51ca66f7ee3c:42089 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42089 2024-11-11T20:52:23,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/51ca66f7ee3c,42089,1731358338732 2024-11-11T20:52:23,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T20:52:23,360 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T20:52:23,362 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [51ca66f7ee3c,42089,1731358338732] 2024-11-11T20:52:23,363 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/51ca66f7ee3c,42089,1731358338732 already deleted, retry=false 2024-11-11T20:52:23,363 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 51ca66f7ee3c,42089,1731358338732 expired; onlineServers=0 2024-11-11T20:52:23,363 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '51ca66f7ee3c,39535,1731358337804' ***** 2024-11-11T20:52:23,363 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T20:52:23,363 INFO [M:0;51ca66f7ee3c:39535 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T20:52:23,363 INFO [M:0;51ca66f7ee3c:39535 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T20:52:23,363 DEBUG [M:0;51ca66f7ee3c:39535 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T20:52:23,364 DEBUG [M:0;51ca66f7ee3c:39535 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T20:52:23,364 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T20:52:23,364 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.large.0-1731358339691 {}] cleaner.HFileCleaner(306): Exit Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.large.0-1731358339691,5,FailOnTimeoutGroup] 2024-11-11T20:52:23,364 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.small.0-1731358339693 {}] cleaner.HFileCleaner(306): Exit Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.small.0-1731358339693,5,FailOnTimeoutGroup] 2024-11-11T20:52:23,364 INFO [M:0;51ca66f7ee3c:39535 {}] hbase.ChoreService(370): Chore service for: master/51ca66f7ee3c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T20:52:23,364 INFO [M:0;51ca66f7ee3c:39535 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T20:52:23,364 DEBUG [M:0;51ca66f7ee3c:39535 {}] master.HMaster(1795): Stopping service threads 2024-11-11T20:52:23,364 INFO [M:0;51ca66f7ee3c:39535 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T20:52:23,365 INFO [M:0;51ca66f7ee3c:39535 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:52:23,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T20:52:23,365 INFO [M:0;51ca66f7ee3c:39535 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:52:23,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,365 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T20:52:23,366 DEBUG [M:0;51ca66f7ee3c:39535 {}] zookeeper.ZKUtil(347): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T20:52:23,366 WARN [M:0;51ca66f7ee3c:39535 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T20:52:23,367 INFO [M:0;51ca66f7ee3c:39535 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/.lastflushedseqids 2024-11-11T20:52:23,377 INFO [M:0;51ca66f7ee3c:39535 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:52:23,377 INFO [M:0;51ca66f7ee3c:39535 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T20:52:23,378 DEBUG [M:0;51ca66f7ee3c:39535 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T20:52:23,378 INFO [M:0;51ca66f7ee3c:39535 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:23,378 DEBUG [M:0;51ca66f7ee3c:39535 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:23,378 DEBUG [M:0;51ca66f7ee3c:39535 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T20:52:23,378 DEBUG [M:0;51ca66f7ee3c:39535 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:23,378 INFO [M:0;51ca66f7ee3c:39535 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-11T20:52:23,387 DEBUG [M:0;51ca66f7ee3c:39535 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 29c51b9891094eeaa4c3eca338230138 2024-11-11T20:52:23,399 DEBUG [M:0;51ca66f7ee3c:39535 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/29c51b9891094eeaa4c3eca338230138 is 82, key is hbase:meta,,1/info:regioninfo/1731358340492/Put/seqid=0 2024-11-11T20:52:23,400 INFO [M:0;51ca66f7ee3c:39535 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/29c51b9891094eeaa4c3eca338230138 2024-11-11T20:52:23,407 DEBUG [M:0;51ca66f7ee3c:39535 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 64e78487e51b40f78d7f35dc1e7fffdb 2024-11-11T20:52:23,419 DEBUG [M:0;51ca66f7ee3c:39535 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/64e78487e51b40f78d7f35dc1e7fffdb is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731358341182/Put/seqid=0 2024-11-11T20:52:23,420 INFO [M:0;51ca66f7ee3c:39535 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/64e78487e51b40f78d7f35dc1e7fffdb 2024-11-11T20:52:23,427 DEBUG [M:0;51ca66f7ee3c:39535 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c74a2ed0102e407db27dabf94488db10 2024-11-11T20:52:23,438 DEBUG [M:0;51ca66f7ee3c:39535 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c74a2ed0102e407db27dabf94488db10 is 69, key is 51ca66f7ee3c,42089,1731358338732/rs:state/1731358339706/Put/seqid=0 2024-11-11T20:52:23,438 INFO [M:0;51ca66f7ee3c:39535 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c74a2ed0102e407db27dabf94488db10 2024-11-11T20:52:23,440 DEBUG [M:0;51ca66f7ee3c:39535 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/29c51b9891094eeaa4c3eca338230138 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/29c51b9891094eeaa4c3eca338230138 2024-11-11T20:52:23,442 INFO [M:0;51ca66f7ee3c:39535 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/29c51b9891094eeaa4c3eca338230138, entries=8, sequenceid=84, filesize=5.5 K 2024-11-11T20:52:23,442 DEBUG [M:0;51ca66f7ee3c:39535 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/64e78487e51b40f78d7f35dc1e7fffdb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/64e78487e51b40f78d7f35dc1e7fffdb 2024-11-11T20:52:23,444 INFO [M:0;51ca66f7ee3c:39535 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/64e78487e51b40f78d7f35dc1e7fffdb, entries=9, sequenceid=84, filesize=6.5 K 2024-11-11T20:52:23,444 DEBUG [M:0;51ca66f7ee3c:39535 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c74a2ed0102e407db27dabf94488db10 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c74a2ed0102e407db27dabf94488db10 2024-11-11T20:52:23,445 INFO [M:0;51ca66f7ee3c:39535 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c74a2ed0102e407db27dabf94488db10, entries=2, sequenceid=84, filesize=5.1 K 2024-11-11T20:52:23,446 INFO [M:0;51ca66f7ee3c:39535 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=84, compaction requested=false 2024-11-11T20:52:23,448 INFO [M:0;51ca66f7ee3c:39535 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:23,448 DEBUG [M:0;51ca66f7ee3c:39535 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731358343378Disabling compacts and flushes for region at 1731358343378Disabling writes for close at 1731358343378Obtaining lock to block concurrent updates at 1731358343378Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731358343378Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731358343379 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731358343379Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731358343379Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731358343398 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731358343398Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731358343401 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731358343419 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731358343419Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731358343422 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731358343437 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731358343437Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66e63133: reopening flushed file at 1731358343440 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64871e11: reopening flushed file at 1731358343442 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a0a575c: reopening flushed file at 1731358343444 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=84, compaction requested=false at 1731358343446 (+2 ms)Writing region close event to WAL at 1731358343448 (+2 ms)Closed at 1731358343448 2024-11-11T20:52:23,450 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T20:52:23,450 INFO [M:0;51ca66f7ee3c:39535 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T20:52:23,450 INFO [M:0;51ca66f7ee3c:39535 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39535 2024-11-11T20:52:23,450 INFO [M:0;51ca66f7ee3c:39535 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T20:52:23,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:52:23,462 INFO [RS:1;51ca66f7ee3c:42089 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T20:52:23,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42089-0x100308f05e40002, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:52:23,462 INFO [RS:1;51ca66f7ee3c:42089 {}] regionserver.HRegionServer(1031): Exiting; stopping=51ca66f7ee3c,42089,1731358338732; zookeeper connection closed. 2024-11-11T20:52:23,462 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@78b725c0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@78b725c0 2024-11-11T20:52:23,462 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-11T20:52:23,552 INFO [M:0;51ca66f7ee3c:39535 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T20:52:23,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:52:23,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39535-0x100308f05e40000, quorum=127.0.0.1:59135, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:52:23,559 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T20:52:23,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-11T20:52:23,578 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:23,585 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/51ca66f7ee3c:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731358343224 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/51ca66f7ee3c:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/51ca66f7ee3c:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1731358343227 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731358343093 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731358341542 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/51ca66f7ee3c:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=240 (was 226) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5763 (was 6199) 2024-11-11T20:52:23,590 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=240, ProcessCount=11, AvailableMemoryMB=5763 2024-11-11T20:52:23,590 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/hadoop-log-dir so I do NOT create it in target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f 2024-11-11T20:52:23,590 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/hadoop-tmp-dir so I do NOT create it in target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f 2024-11-11T20:52:23,590 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:23,593 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/cluster_86cd97e5-c77c-81b5-d9ea-99f58bf8bc4a, deleteOnExit=true 2024-11-11T20:52:23,595 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/cluster_86cd97e5-c77c-81b5-d9ea-99f58bf8bc4a/zookeeper_0, clientPort=60875, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/cluster_86cd97e5-c77c-81b5-d9ea-99f58bf8bc4a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/cluster_86cd97e5-c77c-81b5-d9ea-99f58bf8bc4a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T20:52:23,596 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60875 2024-11-11T20:52:23,610 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8 with version=8 2024-11-11T20:52:23,610 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/hbase-staging 2024-11-11T20:52:23,613 INFO [Time-limited test {}] client.ConnectionUtils(128): master/51ca66f7ee3c:0 server-side Connection retries=45 2024-11-11T20:52:23,613 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:23,613 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:23,613 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T20:52:23,613 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:23,613 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T20:52:23,613 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T20:52:23,614 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T20:52:23,615 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41949 2024-11-11T20:52:23,617 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41949 connecting to ZooKeeper ensemble=127.0.0.1:60875 2024-11-11T20:52:23,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:419490x0, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T20:52:23,626 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41949-0x100308f1e190000 connected 2024-11-11T20:52:23,672 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:23,672 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:23,674 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:52:23,675 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8, hbase.cluster.distributed=false 2024-11-11T20:52:23,678 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T20:52:23,679 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41949 2024-11-11T20:52:23,679 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41949 2024-11-11T20:52:23,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41949 2024-11-11T20:52:23,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41949 2024-11-11T20:52:23,682 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41949 2024-11-11T20:52:23,701 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/51ca66f7ee3c:0 server-side Connection retries=45 2024-11-11T20:52:23,701 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:23,701 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:23,701 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T20:52:23,701 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:23,701 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T20:52:23,701 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T20:52:23,701 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T20:52:23,709 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41961 2024-11-11T20:52:23,712 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41961 connecting to ZooKeeper ensemble=127.0.0.1:60875 2024-11-11T20:52:23,713 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:23,713 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:23,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:419610x0, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T20:52:23,720 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41961-0x100308f1e190001 connected 2024-11-11T20:52:23,722 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:52:23,722 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T20:52:23,724 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/bucket.cache 2024-11-11T20:52:23,724 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-11T20:52:23,725 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-11T20:52:23,725 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-11T20:52:23,726 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-11T20:52:23,726 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-11T20:52:23,727 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-11T20:52:23,727 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/bucket.persistence is complete. 2024-11-11T20:52:23,732 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-11T20:52:23,734 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T20:52:23,735 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T20:52:23,736 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T20:52:23,737 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41961 2024-11-11T20:52:23,738 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41961 2024-11-11T20:52:23,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41961 2024-11-11T20:52:23,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41961 2024-11-11T20:52:23,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41961 2024-11-11T20:52:23,758 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/51ca66f7ee3c:0 server-side Connection retries=45 2024-11-11T20:52:23,758 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:23,759 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:23,759 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T20:52:23,759 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:23,759 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T20:52:23,759 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T20:52:23,759 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T20:52:23,760 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34157 2024-11-11T20:52:23,761 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34157 connecting to ZooKeeper ensemble=127.0.0.1:60875 2024-11-11T20:52:23,762 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:23,762 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:23,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:341570x0, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T20:52:23,766 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:52:23,766 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34157-0x100308f1e190002 connected 2024-11-11T20:52:23,766 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T20:52:23,767 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/bucket.cache 2024-11-11T20:52:23,768 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-11T20:52:23,768 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-11T20:52:23,768 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-11T20:52:23,770 INFO [Thread-121 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-11T20:52:23,770 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-11T20:52:23,770 WARN [Thread-121 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-11T20:52:23,771 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-11T20:52:23,771 INFO [Thread-121 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/bucket.persistence is complete. 2024-11-11T20:52:23,773 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T20:52:23,774 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T20:52:23,775 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T20:52:23,776 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34157 2024-11-11T20:52:23,777 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34157 2024-11-11T20:52:23,780 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34157 2024-11-11T20:52:23,784 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34157 2024-11-11T20:52:23,785 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34157 2024-11-11T20:52:23,801 DEBUG [M:0;51ca66f7ee3c:41949 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;51ca66f7ee3c:41949 2024-11-11T20:52:23,801 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/51ca66f7ee3c,41949,1731358343612 2024-11-11T20:52:23,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:52:23,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:52:23,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:52:23,804 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/51ca66f7ee3c,41949,1731358343612 2024-11-11T20:52:23,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T20:52:23,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T20:52:23,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,806 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T20:52:23,806 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/51ca66f7ee3c,41949,1731358343612 from backup master directory 2024-11-11T20:52:23,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/51ca66f7ee3c,41949,1731358343612 2024-11-11T20:52:23,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:52:23,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:52:23,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:52:23,807 WARN [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T20:52:23,807 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=51ca66f7ee3c,41949,1731358343612 2024-11-11T20:52:23,808 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/hbase.id] with ID: 9748f694-2e2c-4e08-a196-fd116876312e 2024-11-11T20:52:23,808 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/.tmp/hbase.id 2024-11-11T20:52:23,813 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/hbase.id] 2024-11-11T20:52:23,843 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:23,844 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T20:52:23,846 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-11T20:52:23,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,852 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:52:23,852 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T20:52:23,853 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:52:23,869 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store 2024-11-11T20:52:23,875 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:23,875 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T20:52:23,875 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:23,875 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:23,875 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T20:52:23,875 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:23,875 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:23,875 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731358343875Disabling compacts and flushes for region at 1731358343875Disabling writes for close at 1731358343875Writing region close event to WAL at 1731358343875Closed at 1731358343875 2024-11-11T20:52:23,875 WARN [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/.initializing 2024-11-11T20:52:23,875 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/WALs/51ca66f7ee3c,41949,1731358343612 2024-11-11T20:52:23,880 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T20:52:23,883 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51ca66f7ee3c%2C41949%2C1731358343612, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/WALs/51ca66f7ee3c,41949,1731358343612, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/oldWALs, maxLogs=10 2024-11-11T20:52:23,899 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/WALs/51ca66f7ee3c,41949,1731358343612/51ca66f7ee3c%2C41949%2C1731358343612.1731358343884 2024-11-11T20:52:23,899 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T20:52:23,899 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:52:23,899 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:23,899 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:23,899 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:23,900 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:23,904 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T20:52:23,905 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:23,905 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:23,905 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:23,908 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T20:52:23,908 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:23,908 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:23,908 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:23,911 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T20:52:23,911 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:23,912 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:23,912 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:23,915 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T20:52:23,916 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:23,916 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:23,916 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:23,916 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:23,916 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:23,917 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:23,917 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:23,917 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T20:52:23,917 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:23,922 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:52:23,922 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74641419, jitterRate=0.11224381625652313}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T20:52:23,923 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731358343900Initializing all the Stores at 1731358343900Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358343900Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358343900Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358343900Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358343900Cleaning up temporary data from old regions at 1731358343917 (+17 ms)Region opened successfully at 1731358343923 (+6 ms) 2024-11-11T20:52:23,923 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T20:52:23,927 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d19c707, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51ca66f7ee3c/172.17.0.2:0 2024-11-11T20:52:23,928 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T20:52:23,928 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T20:52:23,928 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T20:52:23,928 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:52:23,928 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:52:23,929 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-11T20:52:23,929 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T20:52:23,931 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T20:52:23,932 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T20:52:23,933 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T20:52:23,933 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T20:52:23,934 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T20:52:23,934 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T20:52:23,935 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T20:52:23,936 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T20:52:23,936 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T20:52:23,937 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T20:52:23,938 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T20:52:23,940 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T20:52:23,941 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T20:52:23,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T20:52:23,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T20:52:23,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T20:52:23,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,944 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=51ca66f7ee3c,41949,1731358343612, sessionid=0x100308f1e190000, setting cluster-up flag (Was=false) 2024-11-11T20:52:23,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,950 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T20:52:23,952 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=51ca66f7ee3c,41949,1731358343612 2024-11-11T20:52:23,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:23,959 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T20:52:23,960 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=51ca66f7ee3c,41949,1731358343612 2024-11-11T20:52:23,961 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T20:52:23,962 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:52:23,963 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T20:52:23,963 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T20:52:23,963 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 51ca66f7ee3c,41949,1731358343612 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T20:52:23,965 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:52:23,965 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:52:23,965 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:52:23,965 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:52:23,965 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/51ca66f7ee3c:0, corePoolSize=10, maxPoolSize=10 2024-11-11T20:52:23,966 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:23,966 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=2, maxPoolSize=2 2024-11-11T20:52:23,966 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:23,969 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358373969 2024-11-11T20:52:23,970 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T20:52:23,970 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T20:52:23,970 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T20:52:23,970 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T20:52:23,970 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T20:52:23,970 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T20:52:23,970 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T20:52:23,970 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T20:52:23,970 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:23,971 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:23,971 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T20:52:23,971 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T20:52:23,972 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T20:52:23,972 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T20:52:23,972 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T20:52:23,972 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T20:52:23,972 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.large.0-1731358343972,5,FailOnTimeoutGroup] 2024-11-11T20:52:23,973 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.small.0-1731358343972,5,FailOnTimeoutGroup] 2024-11-11T20:52:23,973 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:23,973 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T20:52:23,973 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:23,973 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:23,985 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T20:52:23,985 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8 2024-11-11T20:52:23,987 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer(746): ClusterId : 9748f694-2e2c-4e08-a196-fd116876312e 2024-11-11T20:52:23,987 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer(746): ClusterId : 9748f694-2e2c-4e08-a196-fd116876312e 2024-11-11T20:52:23,987 DEBUG [RS:0;51ca66f7ee3c:41961 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T20:52:23,987 DEBUG [RS:1;51ca66f7ee3c:34157 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T20:52:23,989 DEBUG [RS:0;51ca66f7ee3c:41961 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T20:52:23,989 DEBUG [RS:1;51ca66f7ee3c:34157 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T20:52:23,989 DEBUG [RS:0;51ca66f7ee3c:41961 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T20:52:23,989 DEBUG [RS:1;51ca66f7ee3c:34157 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T20:52:23,992 DEBUG [RS:1;51ca66f7ee3c:34157 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T20:52:23,992 DEBUG [RS:0;51ca66f7ee3c:41961 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T20:52:23,992 DEBUG [RS:0;51ca66f7ee3c:41961 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab6acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51ca66f7ee3c/172.17.0.2:0 2024-11-11T20:52:23,992 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:23,992 DEBUG [RS:1;51ca66f7ee3c:34157 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43236a84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51ca66f7ee3c/172.17.0.2:0 2024-11-11T20:52:23,997 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-11T20:52:24,005 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T20:52:24,006 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:24,006 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:24,006 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-11T20:52:24,008 DEBUG [RS:0;51ca66f7ee3c:41961 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;51ca66f7ee3c:41961 2024-11-11T20:52:24,009 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T20:52:24,009 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T20:52:24,009 DEBUG [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T20:52:24,010 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T20:52:24,010 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:24,010 DEBUG [RS:1;51ca66f7ee3c:34157 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;51ca66f7ee3c:34157 2024-11-11T20:52:24,010 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T20:52:24,010 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T20:52:24,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:24,010 DEBUG [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T20:52:24,010 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer(2659): reportForDuty to master=51ca66f7ee3c,41949,1731358343612 with port=41961, startcode=1731358343700 2024-11-11T20:52:24,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-11T20:52:24,010 DEBUG [RS:0;51ca66f7ee3c:41961 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T20:52:24,011 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer(2659): reportForDuty to master=51ca66f7ee3c,41949,1731358343612 with port=34157, startcode=1731358343758 2024-11-11T20:52:24,011 DEBUG [RS:1;51ca66f7ee3c:34157 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T20:52:24,013 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39611, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T20:52:24,014 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41949 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 51ca66f7ee3c,41961,1731358343700 2024-11-11T20:52:24,014 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41949 {}] master.ServerManager(517): Registering regionserver=51ca66f7ee3c,41961,1731358343700 2024-11-11T20:52:24,016 DEBUG [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8 2024-11-11T20:52:24,016 DEBUG [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-11T20:52:24,016 DEBUG [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T20:52:24,018 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44143, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T20:52:24,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T20:52:24,019 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41949 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 51ca66f7ee3c,34157,1731358343758 2024-11-11T20:52:24,019 DEBUG [RS:0;51ca66f7ee3c:41961 {}] zookeeper.ZKUtil(111): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/51ca66f7ee3c,41961,1731358343700 2024-11-11T20:52:24,019 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41949 {}] master.ServerManager(517): Registering regionserver=51ca66f7ee3c,34157,1731358343758 2024-11-11T20:52:24,019 WARN [RS:0;51ca66f7ee3c:41961 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T20:52:24,019 INFO [RS:0;51ca66f7ee3c:41961 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:52:24,019 DEBUG [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,41961,1731358343700 2024-11-11T20:52:24,021 DEBUG [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8 2024-11-11T20:52:24,021 DEBUG [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-11T20:52:24,021 DEBUG [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T20:52:24,022 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T20:52:24,022 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:24,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:24,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-11T20:52:24,024 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [51ca66f7ee3c,34157,1731358343758] 2024-11-11T20:52:24,024 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [51ca66f7ee3c,41961,1731358343700] 2024-11-11T20:52:24,024 DEBUG [RS:1;51ca66f7ee3c:34157 {}] zookeeper.ZKUtil(111): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/51ca66f7ee3c,34157,1731358343758 2024-11-11T20:52:24,024 WARN [RS:1;51ca66f7ee3c:34157 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T20:52:24,024 INFO [RS:1;51ca66f7ee3c:34157 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:52:24,025 DEBUG [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,34157,1731358343758 2024-11-11T20:52:24,031 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T20:52:24,032 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:24,032 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:24,032 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T20:52:24,033 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740 2024-11-11T20:52:24,033 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T20:52:24,033 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740 2024-11-11T20:52:24,033 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T20:52:24,034 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T20:52:24,034 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T20:52:24,035 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T20:52:24,035 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T20:52:24,036 INFO [RS:0;51ca66f7ee3c:41961 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T20:52:24,036 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,036 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T20:52:24,041 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T20:52:24,041 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T20:52:24,042 INFO [RS:1;51ca66f7ee3c:34157 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T20:52:24,042 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T20:52:24,042 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,042 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,042 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T20:52:24,042 DEBUG [RS:0;51ca66f7ee3c:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,042 DEBUG [RS:0;51ca66f7ee3c:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,043 DEBUG [RS:0;51ca66f7ee3c:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,043 DEBUG [RS:0;51ca66f7ee3c:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,043 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T20:52:24,043 DEBUG [RS:0;51ca66f7ee3c:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,043 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,043 DEBUG [RS:0;51ca66f7ee3c:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=2, maxPoolSize=2 2024-11-11T20:52:24,043 DEBUG [RS:0;51ca66f7ee3c:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,043 DEBUG [RS:1;51ca66f7ee3c:34157 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,043 DEBUG [RS:0;51ca66f7ee3c:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,043 DEBUG [RS:1;51ca66f7ee3c:34157 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,043 DEBUG [RS:0;51ca66f7ee3c:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,043 DEBUG [RS:1;51ca66f7ee3c:34157 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,043 DEBUG [RS:0;51ca66f7ee3c:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,044 DEBUG [RS:1;51ca66f7ee3c:34157 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,044 DEBUG [RS:0;51ca66f7ee3c:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,044 DEBUG [RS:1;51ca66f7ee3c:34157 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,044 DEBUG [RS:0;51ca66f7ee3c:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,044 DEBUG [RS:1;51ca66f7ee3c:34157 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=2, maxPoolSize=2 2024-11-11T20:52:24,044 DEBUG [RS:0;51ca66f7ee3c:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:52:24,044 DEBUG [RS:0;51ca66f7ee3c:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:52:24,044 DEBUG [RS:1;51ca66f7ee3c:34157 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,044 DEBUG [RS:1;51ca66f7ee3c:34157 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,044 DEBUG [RS:1;51ca66f7ee3c:34157 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,044 DEBUG [RS:1;51ca66f7ee3c:34157 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,044 DEBUG [RS:1;51ca66f7ee3c:34157 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,044 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,045 DEBUG [RS:1;51ca66f7ee3c:34157 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:24,045 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,045 DEBUG [RS:1;51ca66f7ee3c:34157 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:52:24,045 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,045 DEBUG [RS:1;51ca66f7ee3c:34157 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:52:24,045 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,045 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,045 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,41961,1731358343700-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T20:52:24,045 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:52:24,045 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74861214, jitterRate=0.11551901698112488}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T20:52:24,046 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731358343993Initializing all the Stores at 1731358343993Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358343993Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358343997 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358343997Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358343997Cleaning up temporary data from old regions at 1731358344034 (+37 ms)Region opened successfully at 1731358344046 (+12 ms) 2024-11-11T20:52:24,046 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T20:52:24,046 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T20:52:24,046 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T20:52:24,046 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T20:52:24,046 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T20:52:24,049 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,049 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,049 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T20:52:24,049 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,049 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731358344046Disabling compacts and flushes for region at 1731358344046Disabling writes for close at 1731358344046Writing region close event to WAL at 1731358344049 (+3 ms)Closed at 1731358344049 2024-11-11T20:52:24,049 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,049 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,049 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,34157,1731358343758-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T20:52:24,050 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T20:52:24,050 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T20:52:24,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T20:52:24,051 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:52:24,052 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T20:52:24,069 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T20:52:24,069 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,41961,1731358343700-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,069 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,069 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.Replication(171): 51ca66f7ee3c,41961,1731358343700 started 2024-11-11T20:52:24,073 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T20:52:24,073 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,34157,1731358343758-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,074 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,074 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.Replication(171): 51ca66f7ee3c,34157,1731358343758 started 2024-11-11T20:52:24,089 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,090 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer(1482): Serving as 51ca66f7ee3c,41961,1731358343700, RpcServer on 51ca66f7ee3c/172.17.0.2:41961, sessionid=0x100308f1e190001 2024-11-11T20:52:24,090 DEBUG [RS:0;51ca66f7ee3c:41961 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T20:52:24,090 DEBUG [RS:0;51ca66f7ee3c:41961 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 51ca66f7ee3c,41961,1731358343700 2024-11-11T20:52:24,090 DEBUG [RS:0;51ca66f7ee3c:41961 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,41961,1731358343700' 2024-11-11T20:52:24,090 DEBUG [RS:0;51ca66f7ee3c:41961 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T20:52:24,091 DEBUG [RS:0;51ca66f7ee3c:41961 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T20:52:24,091 DEBUG [RS:0;51ca66f7ee3c:41961 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T20:52:24,091 DEBUG [RS:0;51ca66f7ee3c:41961 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T20:52:24,091 DEBUG [RS:0;51ca66f7ee3c:41961 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 51ca66f7ee3c,41961,1731358343700 2024-11-11T20:52:24,091 DEBUG [RS:0;51ca66f7ee3c:41961 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,41961,1731358343700' 2024-11-11T20:52:24,091 DEBUG [RS:0;51ca66f7ee3c:41961 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T20:52:24,092 DEBUG [RS:0;51ca66f7ee3c:41961 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T20:52:24,092 DEBUG [RS:0;51ca66f7ee3c:41961 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T20:52:24,092 INFO [RS:0;51ca66f7ee3c:41961 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T20:52:24,092 INFO [RS:0;51ca66f7ee3c:41961 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T20:52:24,093 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,093 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer(1482): Serving as 51ca66f7ee3c,34157,1731358343758, RpcServer on 51ca66f7ee3c/172.17.0.2:34157, sessionid=0x100308f1e190002 2024-11-11T20:52:24,093 DEBUG [RS:1;51ca66f7ee3c:34157 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T20:52:24,094 DEBUG [RS:1;51ca66f7ee3c:34157 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 51ca66f7ee3c,34157,1731358343758 2024-11-11T20:52:24,094 DEBUG [RS:1;51ca66f7ee3c:34157 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,34157,1731358343758' 2024-11-11T20:52:24,094 DEBUG [RS:1;51ca66f7ee3c:34157 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T20:52:24,094 DEBUG [RS:1;51ca66f7ee3c:34157 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T20:52:24,095 DEBUG [RS:1;51ca66f7ee3c:34157 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T20:52:24,095 DEBUG [RS:1;51ca66f7ee3c:34157 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T20:52:24,095 DEBUG [RS:1;51ca66f7ee3c:34157 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 51ca66f7ee3c,34157,1731358343758 2024-11-11T20:52:24,095 DEBUG [RS:1;51ca66f7ee3c:34157 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,34157,1731358343758' 2024-11-11T20:52:24,095 DEBUG [RS:1;51ca66f7ee3c:34157 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T20:52:24,095 DEBUG [RS:1;51ca66f7ee3c:34157 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T20:52:24,095 DEBUG [RS:1;51ca66f7ee3c:34157 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T20:52:24,096 INFO [RS:1;51ca66f7ee3c:34157 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T20:52:24,096 INFO [RS:1;51ca66f7ee3c:34157 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T20:52:24,194 INFO [RS:0;51ca66f7ee3c:41961 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T20:52:24,195 INFO [RS:0;51ca66f7ee3c:41961 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=51ca66f7ee3c%2C41961%2C1731358343700, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,41961,1731358343700, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/oldWALs, maxLogs=55 2024-11-11T20:52:24,196 INFO [RS:1;51ca66f7ee3c:34157 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T20:52:24,198 INFO [RS:1;51ca66f7ee3c:34157 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=51ca66f7ee3c%2C34157%2C1731358343758, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,34157,1731358343758, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/oldWALs, maxLogs=55 2024-11-11T20:52:24,203 WARN [51ca66f7ee3c:41949 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T20:52:24,218 INFO [RS:0;51ca66f7ee3c:41961 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,41961,1731358343700/51ca66f7ee3c%2C41961%2C1731358343700.1731358344202 2024-11-11T20:52:24,218 INFO [RS:1;51ca66f7ee3c:34157 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,34157,1731358343758/51ca66f7ee3c%2C34157%2C1731358343758.1731358344202 2024-11-11T20:52:24,219 DEBUG [RS:0;51ca66f7ee3c:41961 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T20:52:24,219 DEBUG [RS:1;51ca66f7ee3c:34157 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T20:52:24,453 DEBUG [51ca66f7ee3c:41949 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-11T20:52:24,454 DEBUG [51ca66f7ee3c:41949 {}] balancer.BalancerClusterState(204): Hosts are {51ca66f7ee3c=0} racks are {/default-rack=0} 2024-11-11T20:52:24,455 DEBUG [51ca66f7ee3c:41949 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T20:52:24,455 DEBUG [51ca66f7ee3c:41949 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T20:52:24,455 DEBUG [51ca66f7ee3c:41949 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T20:52:24,455 DEBUG [51ca66f7ee3c:41949 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T20:52:24,455 INFO [51ca66f7ee3c:41949 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T20:52:24,455 INFO [51ca66f7ee3c:41949 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T20:52:24,455 DEBUG [51ca66f7ee3c:41949 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T20:52:24,455 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=51ca66f7ee3c,34157,1731358343758 2024-11-11T20:52:24,457 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,34157,1731358343758, state=OPENING 2024-11-11T20:52:24,458 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T20:52:24,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:24,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:24,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:24,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:24,461 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:52:24,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:24,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:24,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,34157,1731358343758}] 2024-11-11T20:52:24,615 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T20:52:24,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53281, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T20:52:24,624 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T20:52:24,625 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:52:24,625 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T20:52:24,626 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=51ca66f7ee3c%2C34157%2C1731358343758.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,34157,1731358343758, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/oldWALs, maxLogs=55 2024-11-11T20:52:24,645 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,34157,1731358343758/51ca66f7ee3c%2C34157%2C1731358343758.meta.1731358344627.meta 2024-11-11T20:52:24,645 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T20:52:24,646 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:52:24,646 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T20:52:24,646 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T20:52:24,646 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T20:52:24,646 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T20:52:24,646 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:24,647 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T20:52:24,647 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T20:52:24,649 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-11T20:52:24,650 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T20:52:24,650 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:24,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:24,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-11T20:52:24,651 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T20:52:24,651 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:24,651 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:24,651 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-11T20:52:24,652 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T20:52:24,652 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:24,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:24,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-11T20:52:24,653 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T20:52:24,653 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:24,653 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:24,653 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T20:52:24,654 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740 2024-11-11T20:52:24,654 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740 2024-11-11T20:52:24,655 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T20:52:24,655 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T20:52:24,655 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T20:52:24,656 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T20:52:24,656 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67063488, jitterRate=-6.761550903320312E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T20:52:24,656 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T20:52:24,657 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731358344647Writing region info on filesystem at 1731358344647Initializing all the Stores at 1731358344647Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358344647Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358344648 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358344648Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358344648Cleaning up temporary data from old regions at 1731358344655 (+7 ms)Running coprocessor post-open hooks at 1731358344657 (+2 ms)Region opened successfully at 1731358344657 2024-11-11T20:52:24,658 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731358344615 2024-11-11T20:52:24,661 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T20:52:24,661 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T20:52:24,661 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,34157,1731358343758 2024-11-11T20:52:24,662 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,34157,1731358343758, state=OPEN 2024-11-11T20:52:24,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:52:24,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:52:24,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:52:24,663 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,34157,1731358343758 2024-11-11T20:52:24,663 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:24,663 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:24,663 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:24,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T20:52:24,666 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,34157,1731358343758 in 202 msec 2024-11-11T20:52:24,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T20:52:24,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 616 msec 2024-11-11T20:52:24,668 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T20:52:24,668 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T20:52:24,671 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:24,671 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,34157,1731358343758, seqNum=-1] 2024-11-11T20:52:24,672 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:24,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44825, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:24,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 716 msec 2024-11-11T20:52:24,678 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731358344678, completionTime=-1 2024-11-11T20:52:24,679 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-11T20:52:24,679 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:52:24,681 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:52:24,681 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358404681 2024-11-11T20:52:24,681 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358464681 2024-11-11T20:52:24,681 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-11T20:52:24,681 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,41949,1731358343612-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,682 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,41949,1731358343612-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,682 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,41949,1731358343612-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,682 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-51ca66f7ee3c:41949, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,682 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,682 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,685 DEBUG [master/51ca66f7ee3c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T20:52:24,687 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.880sec 2024-11-11T20:52:24,687 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T20:52:24,687 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T20:52:24,687 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T20:52:24,688 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T20:52:24,688 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T20:52:24,688 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,41949,1731358343612-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T20:52:24,688 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,41949,1731358343612-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T20:52:24,689 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T20:52:24,689 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T20:52:24,689 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,41949,1731358343612-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:24,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc06048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:24,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,41949,-1 for getting cluster id 2024-11-11T20:52:24,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:52:24,792 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9748f694-2e2c-4e08-a196-fd116876312e' 2024-11-11T20:52:24,792 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:52:24,793 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9748f694-2e2c-4e08-a196-fd116876312e" 2024-11-11T20:52:24,794 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@748b46aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:24,794 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,41949,-1] 2024-11-11T20:52:24,795 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:52:24,795 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:24,797 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:52:24,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:24,799 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:24,801 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,34157,1731358343758, seqNum=-1] 2024-11-11T20:52:24,802 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:24,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:24,807 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=51ca66f7ee3c,41949,1731358343612 2024-11-11T20:52:24,808 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:52:24,809 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,41949,1731358343612 2024-11-11T20:52:24,809 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@754d162c 2024-11-11T20:52:24,809 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:52:24,812 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:52:24,813 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41949 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731358344807', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:52:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41949 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731358344807 2024-11-11T20:52:24,815 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731358344807 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T20:52:24,815 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:24,815 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41949 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731358344807" procId is: 4 2024-11-11T20:52:24,816 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731358344807 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T20:52:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:52:24,827 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731358344807-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 14831ea77f9af8d5ab1d6eaf567029e1, NAME => 'testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731358344807', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8 2024-11-11T20:52:24,835 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731358344807-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:24,835 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731358344807-pool-0 {}] regionserver.HRegion(1722): Closing 14831ea77f9af8d5ab1d6eaf567029e1, disabling compactions & flushes 2024-11-11T20:52:24,835 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731358344807-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:24,835 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731358344807-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:24,835 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731358344807-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. after waiting 0 ms 2024-11-11T20:52:24,835 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731358344807-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:24,835 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731358344807-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:24,836 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731358344807-pool-0 {}] regionserver.HRegion(1676): Region close journal for 14831ea77f9af8d5ab1d6eaf567029e1: Waiting for close lock at 1731358344835Disabling compacts and flushes for region at 1731358344835Disabling writes for close at 1731358344835Writing region close event to WAL at 1731358344835Closed at 1731358344835 2024-11-11T20:52:24,837 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731358344807 execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T20:52:24,837 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731358344837"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358344837"}]},"ts":"1731358344837"} 2024-11-11T20:52:24,840 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T20:52:24,842 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731358344807 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T20:52:24,842 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731358344807","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358344842"}]},"ts":"1731358344842"} 2024-11-11T20:52:24,845 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731358344807, state=ENABLING in hbase:meta 2024-11-11T20:52:24,845 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {51ca66f7ee3c=0} racks are {/default-rack=0} 2024-11-11T20:52:24,846 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T20:52:24,846 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T20:52:24,846 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T20:52:24,846 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T20:52:24,846 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T20:52:24,846 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T20:52:24,846 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T20:52:24,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731358344807, region=14831ea77f9af8d5ab1d6eaf567029e1, ASSIGN}] 2024-11-11T20:52:24,848 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731358344807, region=14831ea77f9af8d5ab1d6eaf567029e1, ASSIGN 2024-11-11T20:52:24,849 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731358344807, region=14831ea77f9af8d5ab1d6eaf567029e1, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,34157,1731358343758; forceNewPlan=false, retain=false 2024-11-11T20:52:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:52:25,000 INFO [51ca66f7ee3c:41949 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-11T20:52:25,001 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=14831ea77f9af8d5ab1d6eaf567029e1, regionState=OPENING, regionLocation=51ca66f7ee3c,34157,1731358343758 2024-11-11T20:52:25,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731358344807, region=14831ea77f9af8d5ab1d6eaf567029e1, ASSIGN because future has completed 2024-11-11T20:52:25,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 14831ea77f9af8d5ab1d6eaf567029e1, server=51ca66f7ee3c,34157,1731358343758}] 2024-11-11T20:52:25,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:52:25,168 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:25,169 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 14831ea77f9af8d5ab1d6eaf567029e1, NAME => 'testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:52:25,169 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731358344807 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:25,169 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:25,169 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:25,170 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:25,170 INFO [StoreOpener-14831ea77f9af8d5ab1d6eaf567029e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:25,174 INFO [StoreOpener-14831ea77f9af8d5ab1d6eaf567029e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14831ea77f9af8d5ab1d6eaf567029e1 columnFamilyName family 2024-11-11T20:52:25,174 DEBUG [StoreOpener-14831ea77f9af8d5ab1d6eaf567029e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:25,174 INFO [StoreOpener-14831ea77f9af8d5ab1d6eaf567029e1-1 {}] regionserver.HStore(327): Store=14831ea77f9af8d5ab1d6eaf567029e1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:25,175 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:25,175 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:25,175 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:25,176 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:25,176 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:25,176 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:25,183 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:52:25,184 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 14831ea77f9af8d5ab1d6eaf567029e1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74690879, jitterRate=0.11298082768917084}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:52:25,184 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:25,185 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 14831ea77f9af8d5ab1d6eaf567029e1: Running coprocessor pre-open hook at 1731358345170Writing region info on filesystem at 1731358345170Initializing all the Stores at 1731358345170Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358345170Cleaning up temporary data from old regions at 1731358345176 (+6 ms)Running coprocessor post-open hooks at 1731358345184 (+8 ms)Region opened successfully at 1731358345185 (+1 ms) 2024-11-11T20:52:25,186 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1., pid=6, masterSystemTime=1731358345163 2024-11-11T20:52:25,188 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:25,188 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:25,189 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=14831ea77f9af8d5ab1d6eaf567029e1, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,34157,1731358343758 2024-11-11T20:52:25,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 14831ea77f9af8d5ab1d6eaf567029e1, server=51ca66f7ee3c,34157,1731358343758 because future has completed 2024-11-11T20:52:25,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T20:52:25,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 14831ea77f9af8d5ab1d6eaf567029e1, server=51ca66f7ee3c,34157,1731358343758 in 184 msec 2024-11-11T20:52:25,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T20:52:25,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731358344807, region=14831ea77f9af8d5ab1d6eaf567029e1, ASSIGN in 349 msec 2024-11-11T20:52:25,197 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731358344807 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T20:52:25,197 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731358344807","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358345197"}]},"ts":"1731358345197"} 2024-11-11T20:52:25,199 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731358344807, state=ENABLED in hbase:meta 2024-11-11T20:52:25,200 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731358344807 execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T20:52:25,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731358344807 in 387 msec 2024-11-11T20:52:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41949 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:52:25,443 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731358344807 completed 2024-11-11T20:52:25,443 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731358344807 get assigned. Timeout = 60000ms 2024-11-11T20:52:25,444 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T20:52:25,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731358344807 assigned to meta. Checking AM states. 2024-11-11T20:52:25,452 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T20:52:25,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731358344807 assigned. 2024-11-11T20:52:25,456 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731358344807', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1., hostname=51ca66f7ee3c,34157,1731358343758, seqNum=2] 2024-11-11T20:52:25,462 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 14831ea77f9af8d5ab1d6eaf567029e1 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-11T20:52:25,470 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 782ec974e2cc4411b7057dfca5957718 2024-11-11T20:52:25,482 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/.tmp/family/782ec974e2cc4411b7057dfca5957718 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-11T20:52:25,482 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/.tmp/family/782ec974e2cc4411b7057dfca5957718 2024-11-11T20:52:25,485 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/.tmp/family/782ec974e2cc4411b7057dfca5957718 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/family/782ec974e2cc4411b7057dfca5957718 2024-11-11T20:52:25,486 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/family/782ec974e2cc4411b7057dfca5957718, delay=940 ms 2024-11-11T20:52:25,487 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/family/782ec974e2cc4411b7057dfca5957718, entries=2, sequenceid=6, filesize=4.9 K 2024-11-11T20:52:25,488 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 14831ea77f9af8d5ab1d6eaf567029e1 in 25ms, sequenceid=6, compaction requested=false 2024-11-11T20:52:25,488 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731358344807' 2024-11-11T20:52:25,488 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 14831ea77f9af8d5ab1d6eaf567029e1: 2024-11-11T20:52:25,967 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T20:52:26,427 DEBUG [hfile-prefetch-1731358343093 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 782ec974e2cc4411b7057dfca5957718_0. 2024-11-11T20:52:26,428 DEBUG [hfile-prefetch-1731358343093 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 782ec974e2cc4411b7057dfca5957718_117. 2024-11-11T20:52:26,429 DEBUG [hfile-prefetch-1731358343093 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/family/782ec974e2cc4411b7057dfca5957718, with total blocks 2, and data blocks 1 2024-11-11T20:52:26,429 DEBUG [hfile-prefetch-1731358343093 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-11T20:52:26,429 DEBUG [hfile-prefetch-1731358343093 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/family/782ec974e2cc4411b7057dfca5957718, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-11T20:52:26,429 DEBUG [hfile-prefetch-1731358343093 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/family/782ec974e2cc4411b7057dfca5957718. Will sleep 100ms and try the verification again. 2024-11-11T20:52:26,473 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-11T20:52:26,473 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-11T20:52:26,473 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T20:52:26,473 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T20:52:26,480 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-11T20:52:26,480 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731358346473 2024-11-11T20:52:26,480 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-11T20:52:26,489 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;51ca66f7ee3c:34157,5,FailOnTimeoutGroup] 2024-11-11T20:52:26,489 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '51ca66f7ee3c,34157,1731358343758' ***** 2024-11-11T20:52:26,489 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-11T20:52:26,490 INFO [RS:1;51ca66f7ee3c:34157 {}] bucket.BucketCache(1739): Disabling cache 2024-11-11T20:52:26,490 INFO [RS:1;51ca66f7ee3c:34157 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/bucket.persistence 2024-11-11T20:52:26,490 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-11T20:52:26,490 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-11T20:52:26,490 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-11T20:52:26,490 INFO [RS:1;51ca66f7ee3c:34157 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-11T20:52:26,491 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-11T20:52:26,501 DEBUG [RS:1;51ca66f7ee3c:34157 {}] bucket.BucketCache(1488): Thread RS:1;51ca66f7ee3c:34157 started persisting bucket cache to file 2024-11-11T20:52:26,501 DEBUG [RS:1;51ca66f7ee3c:34157 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T20:52:26,501 DEBUG [RS:1;51ca66f7ee3c:34157 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T20:52:26,504 DEBUG [RS:1;51ca66f7ee3c:34157 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-11T20:52:26,504 DEBUG [RS:1;51ca66f7ee3c:34157 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731358346501 2024-11-11T20:52:26,504 DEBUG [RS:1;51ca66f7ee3c:34157 {}] bucket.BucketCache(1510): Thread RS:1;51ca66f7ee3c:34157 finished persisting bucket cache to file, renaming 2024-11-11T20:52:26,504 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T20:52:26,504 INFO [RS:1;51ca66f7ee3c:34157 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T20:52:26,504 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T20:52:26,504 INFO [RS:1;51ca66f7ee3c:34157 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T20:52:26,504 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer(3091): Received CLOSE for 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:26,504 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer(959): stopping server 51ca66f7ee3c,34157,1731358343758 2024-11-11T20:52:26,504 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T20:52:26,504 INFO [RS:1;51ca66f7ee3c:34157 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;51ca66f7ee3c:34157. 2024-11-11T20:52:26,505 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-11T20:52:26,505 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 14831ea77f9af8d5ab1d6eaf567029e1, disabling compactions & flushes 2024-11-11T20:52:26,505 DEBUG [RS:1;51ca66f7ee3c:34157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:52:26,505 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:26,505 DEBUG [RS:1;51ca66f7ee3c:34157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:26,505 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:26,505 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. after waiting 0 ms 2024-11-11T20:52:26,505 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T20:52:26,505 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:26,505 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T20:52:26,505 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T20:52:26,505 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T20:52:26,505 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-11T20:52:26,505 DEBUG [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 14831ea77f9af8d5ab1d6eaf567029e1=testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1.} 2024-11-11T20:52:26,505 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-11T20:52:26,505 DEBUG [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer(1351): Waiting on 14831ea77f9af8d5ab1d6eaf567029e1, 1588230740 2024-11-11T20:52:26,506 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T20:52:26,506 DEBUG [StoreFileCloser-14831ea77f9af8d5ab1d6eaf567029e1-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/family/782ec974e2cc4411b7057dfca5957718 2024-11-11T20:52:26,506 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T20:52:26,506 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T20:52:26,506 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T20:52:26,506 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T20:52:26,506 WARN [hfile-prefetch-1731358343093 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/family/782ec974e2cc4411b7057dfca5957718, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-11T20:52:26,506 DEBUG [hfile-prefetch-1731358343093 {}] hfile.PrefetchExecutor(121): Prefetch completed for 782ec974e2cc4411b7057dfca5957718 2024-11-11T20:52:26,506 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-11T20:52:26,510 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-11T20:52:26,510 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:26,510 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 14831ea77f9af8d5ab1d6eaf567029e1: Waiting for close lock at 1731358346504Running coprocessor pre-close hooks at 1731358346505 (+1 ms)Disabling compacts and flushes for region at 1731358346505Disabling writes for close at 1731358346505Writing region close event to WAL at 1731358346506 (+1 ms)Running coprocessor post-close hooks at 1731358346510 (+4 ms)Closed at 1731358346510 2024-11-11T20:52:26,510 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:26,514 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 50c9486792674700b42a6281367b1d30 2024-11-11T20:52:26,526 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/info/50c9486792674700b42a6281367b1d30 is 209, key is testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1./info:regioninfo/1731358345189/Put/seqid=0 2024-11-11T20:52:26,527 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/info/50c9486792674700b42a6281367b1d30 2024-11-11T20:52:26,535 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 14177cbb01e1428eae69f8fc7c41c980 2024-11-11T20:52:26,548 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/ns/14177cbb01e1428eae69f8fc7c41c980 is 43, key is default/ns:d/1731358344674/Put/seqid=0 2024-11-11T20:52:26,549 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/ns/14177cbb01e1428eae69f8fc7c41c980 2024-11-11T20:52:26,556 INFO [regionserver/51ca66f7ee3c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:52:26,557 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5274cb5b4da043e981f865d5a5096a02 2024-11-11T20:52:26,571 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/table/5274cb5b4da043e981f865d5a5096a02 is 80, key is testBlockEvictionOnGracefulStop1731358344807/table:state/1731358345197/Put/seqid=0 2024-11-11T20:52:26,572 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/table/5274cb5b4da043e981f865d5a5096a02 2024-11-11T20:52:26,575 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/info/50c9486792674700b42a6281367b1d30 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/info/50c9486792674700b42a6281367b1d30 2024-11-11T20:52:26,577 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/info/50c9486792674700b42a6281367b1d30, delay=923 ms 2024-11-11T20:52:26,578 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/info/50c9486792674700b42a6281367b1d30, entries=10, sequenceid=11, filesize=7.0 K 2024-11-11T20:52:26,579 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/ns/14177cbb01e1428eae69f8fc7c41c980 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/ns/14177cbb01e1428eae69f8fc7c41c980 2024-11-11T20:52:26,580 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/ns/14177cbb01e1428eae69f8fc7c41c980, delay=948 ms 2024-11-11T20:52:26,581 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/ns/14177cbb01e1428eae69f8fc7c41c980, entries=2, sequenceid=11, filesize=5.0 K 2024-11-11T20:52:26,582 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/table/5274cb5b4da043e981f865d5a5096a02 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/table/5274cb5b4da043e981f865d5a5096a02 2024-11-11T20:52:26,583 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/table/5274cb5b4da043e981f865d5a5096a02, delay=988 ms 2024-11-11T20:52:26,585 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/table/5274cb5b4da043e981f865d5a5096a02, entries=2, sequenceid=11, filesize=5.3 K 2024-11-11T20:52:26,585 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=11, compaction requested=false 2024-11-11T20:52:26,586 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/info/50c9486792674700b42a6281367b1d30 2024-11-11T20:52:26,587 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/ns/14177cbb01e1428eae69f8fc7c41c980 2024-11-11T20:52:26,587 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/table/5274cb5b4da043e981f865d5a5096a02 2024-11-11T20:52:26,591 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-11T20:52:26,591 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T20:52:26,591 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T20:52:26,591 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731358346505Running coprocessor pre-close hooks at 1731358346505Disabling compacts and flushes for region at 1731358346505Disabling writes for close at 1731358346506 (+1 ms)Obtaining lock to block concurrent updates at 1731358346506Preparing flush snapshotting stores in 1588230740 at 1731358346506Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731358346507 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731358346507Flushing 1588230740/info: creating writer at 1731358346507Flushing 1588230740/info: appending metadata at 1731358346526 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731358346526Flushing 1588230740/ns: creating writer at 1731358346530 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731358346547 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731358346547Flushing 1588230740/table: creating writer at 1731358346552 (+5 ms)Flushing 1588230740/table: appending metadata at 1731358346571 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731358346571Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5eb41db5: reopening flushed file at 1731358346575 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bbc445e: reopening flushed file at 1731358346578 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2472d811: reopening flushed file at 1731358346582 (+4 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=11, compaction requested=false at 1731358346585 (+3 ms)Writing region close event to WAL at 1731358346587 (+2 ms)Running coprocessor post-close hooks at 1731358346591 (+4 ms)Closed at 1731358346591 2024-11-11T20:52:26,591 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T20:52:26,706 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer(976): stopping server 51ca66f7ee3c,34157,1731358343758; all regions closed. 2024-11-11T20:52:26,708 DEBUG [RS:1;51ca66f7ee3c:34157 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/oldWALs 2024-11-11T20:52:26,708 INFO [RS:1;51ca66f7ee3c:34157 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51ca66f7ee3c%2C34157%2C1731358343758.meta:.meta(num 1731358344627) 2024-11-11T20:52:26,710 DEBUG [RS:1;51ca66f7ee3c:34157 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/oldWALs 2024-11-11T20:52:26,710 INFO [RS:1;51ca66f7ee3c:34157 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51ca66f7ee3c%2C34157%2C1731358343758:(num 1731358344202) 2024-11-11T20:52:26,710 DEBUG [RS:1;51ca66f7ee3c:34157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:26,710 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:52:26,710 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T20:52:26,711 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.ChoreService(370): Chore service for: regionserver/51ca66f7ee3c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T20:52:26,711 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T20:52:26,711 INFO [regionserver/51ca66f7ee3c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T20:52:26,711 INFO [RS:1;51ca66f7ee3c:34157 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34157 2024-11-11T20:52:26,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T20:52:26,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/51ca66f7ee3c,34157,1731358343758 2024-11-11T20:52:26,715 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T20:52:26,716 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [51ca66f7ee3c,34157,1731358343758] 2024-11-11T20:52:26,717 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/51ca66f7ee3c,34157,1731358343758 already deleted, retry=false 2024-11-11T20:52:26,718 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 51ca66f7ee3c,34157,1731358343758 on 51ca66f7ee3c,41949,1731358343612 2024-11-11T20:52:26,722 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 51ca66f7ee3c,34157,1731358343758, splitWal=true, meta=true 2024-11-11T20:52:26,725 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 51ca66f7ee3c,34157,1731358343758, splitWal=true, meta=true 2024-11-11T20:52:26,725 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 51ca66f7ee3c,34157,1731358343758 (carryingMeta=true) 51ca66f7ee3c,34157,1731358343758/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@97ea519[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-11T20:52:26,726 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 51ca66f7ee3c,34157,1731358343758, splitWal=true, meta=true, isMeta: true 2024-11-11T20:52:26,726 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,34157,1731358343758-splitting 2024-11-11T20:52:26,727 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,34157,1731358343758-splitting dir is empty, no logs to split. 2024-11-11T20:52:26,727 INFO [PEWorker-1 {}] master.SplitWALManager(105): 51ca66f7ee3c,34157,1731358343758 WAL count=0, meta=true 2024-11-11T20:52:26,729 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,34157,1731358343758-splitting dir is empty, no logs to split. 2024-11-11T20:52:26,729 INFO [PEWorker-1 {}] master.SplitWALManager(105): 51ca66f7ee3c,34157,1731358343758 WAL count=0, meta=true 2024-11-11T20:52:26,729 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 51ca66f7ee3c,34157,1731358343758 WAL splitting is done? wals=0, meta=true 2024-11-11T20:52:26,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T20:52:26,732 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:52:26,733 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T20:52:26,816 INFO [RS:1;51ca66f7ee3c:34157 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T20:52:26,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:52:26,816 INFO [RS:1;51ca66f7ee3c:34157 {}] regionserver.HRegionServer(1031): Exiting; stopping=51ca66f7ee3c,34157,1731358343758; zookeeper connection closed. 2024-11-11T20:52:26,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34157-0x100308f1e190002, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:52:26,817 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1cea69b2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1cea69b2 2024-11-11T20:52:26,884 DEBUG [51ca66f7ee3c:41949 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T20:52:26,884 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=51ca66f7ee3c,41961,1731358343700 2024-11-11T20:52:26,885 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,41961,1731358343700, state=OPENING 2024-11-11T20:52:26,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:52:26,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:52:26,887 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:52:26,887 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:26,887 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:26,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,41961,1731358343700}] 2024-11-11T20:52:27,011 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/51ca66f7ee3c:0 server-side Connection retries=45 2024-11-11T20:52:27,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:27,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:27,011 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T20:52:27,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:27,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T20:52:27,011 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T20:52:27,012 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T20:52:27,012 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46645 2024-11-11T20:52:27,014 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46645 connecting to ZooKeeper ensemble=127.0.0.1:60875 2024-11-11T20:52:27,014 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:27,014 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:27,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:466450x0, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T20:52:27,017 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46645-0x100308f1e190003 connected 2024-11-11T20:52:27,017 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46645-0x100308f1e190003, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-11T20:52:27,017 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-11T20:52:27,018 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T20:52:27,018 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/bucket.cache 2024-11-11T20:52:27,018 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-11T20:52:27,018 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-11T20:52:27,018 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-11T20:52:27,019 INFO [Thread-208 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-11T20:52:27,019 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-11T20:52:27,019 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T20:52:27,020 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46645-0x100308f1e190003, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T20:52:27,020 INFO [Thread-208 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-11T20:52:27,021 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46645-0x100308f1e190003, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T20:52:27,022 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46645 2024-11-11T20:52:27,022 INFO [Thread-208 {}] bucket.BucketCache(1706): Retrieved 1 of chunks with blockCount = 1. 2024-11-11T20:52:27,022 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46645 2024-11-11T20:52:27,022 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46645 2024-11-11T20:52:27,023 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46645 2024-11-11T20:52:27,023 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46645 2024-11-11T20:52:27,024 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.HRegionServer(746): ClusterId : 9748f694-2e2c-4e08-a196-fd116876312e 2024-11-11T20:52:27,024 DEBUG [RS:2;51ca66f7ee3c:46645 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T20:52:27,026 DEBUG [RS:2;51ca66f7ee3c:46645 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T20:52:27,026 DEBUG [RS:2;51ca66f7ee3c:46645 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T20:52:27,027 DEBUG [Thread-208 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-11T20:52:27,028 DEBUG [RS:2;51ca66f7ee3c:46645 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T20:52:27,028 DEBUG [RS:2;51ca66f7ee3c:46645 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24374dce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51ca66f7ee3c/172.17.0.2:0 2024-11-11T20:52:27,028 INFO [Thread-208 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-11T20:52:27,028 INFO [Thread-208 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 1 2024-11-11T20:52:27,028 INFO [Thread-208 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/bucket.persistence is complete. 2024-11-11T20:52:27,038 DEBUG [RS:2;51ca66f7ee3c:46645 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;51ca66f7ee3c:46645 2024-11-11T20:52:27,039 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T20:52:27,039 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T20:52:27,039 DEBUG [RS:2;51ca66f7ee3c:46645 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T20:52:27,040 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.HRegionServer(2659): reportForDuty to master=51ca66f7ee3c,41949,1731358343612 with port=46645, startcode=1731358347011 2024-11-11T20:52:27,040 DEBUG [RS:2;51ca66f7ee3c:46645 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T20:52:27,040 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T20:52:27,042 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46941, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T20:52:27,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33561, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T20:52:27,042 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41949 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 51ca66f7ee3c,46645,1731358347011 2024-11-11T20:52:27,042 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41949 {}] master.ServerManager(517): Registering regionserver=51ca66f7ee3c,46645,1731358347011 2024-11-11T20:52:27,044 DEBUG [RS:2;51ca66f7ee3c:46645 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8 2024-11-11T20:52:27,044 DEBUG [RS:2;51ca66f7ee3c:46645 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-11T20:52:27,044 DEBUG [RS:2;51ca66f7ee3c:46645 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T20:52:27,044 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T20:52:27,044 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:52:27,044 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T20:52:27,045 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=51ca66f7ee3c%2C41961%2C1731358343700.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,41961,1731358343700, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/oldWALs, maxLogs=55 2024-11-11T20:52:27,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T20:52:27,046 DEBUG [RS:2;51ca66f7ee3c:46645 {}] zookeeper.ZKUtil(111): regionserver:46645-0x100308f1e190003, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/51ca66f7ee3c,46645,1731358347011 2024-11-11T20:52:27,046 WARN [RS:2;51ca66f7ee3c:46645 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T20:52:27,046 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [51ca66f7ee3c,46645,1731358347011] 2024-11-11T20:52:27,046 INFO [RS:2;51ca66f7ee3c:46645 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:52:27,046 DEBUG [RS:2;51ca66f7ee3c:46645 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,46645,1731358347011 2024-11-11T20:52:27,052 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T20:52:27,055 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T20:52:27,055 INFO [RS:2;51ca66f7ee3c:46645 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T20:52:27,056 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:27,056 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T20:52:27,056 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T20:52:27,056 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:27,057 DEBUG [RS:2;51ca66f7ee3c:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:27,057 DEBUG [RS:2;51ca66f7ee3c:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:27,057 DEBUG [RS:2;51ca66f7ee3c:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:27,057 DEBUG [RS:2;51ca66f7ee3c:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:27,057 DEBUG [RS:2;51ca66f7ee3c:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:27,057 DEBUG [RS:2;51ca66f7ee3c:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=2, maxPoolSize=2 2024-11-11T20:52:27,057 DEBUG [RS:2;51ca66f7ee3c:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:27,057 DEBUG [RS:2;51ca66f7ee3c:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:27,057 DEBUG [RS:2;51ca66f7ee3c:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:27,057 DEBUG [RS:2;51ca66f7ee3c:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:27,058 DEBUG [RS:2;51ca66f7ee3c:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:27,058 DEBUG [RS:2;51ca66f7ee3c:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:27,058 DEBUG [RS:2;51ca66f7ee3c:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:52:27,058 DEBUG [RS:2;51ca66f7ee3c:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:52:27,058 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:27,058 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:27,058 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:27,058 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:27,058 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:27,059 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,46645,1731358347011-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T20:52:27,066 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,41961,1731358343700/51ca66f7ee3c%2C41961%2C1731358343700.meta.1731358347046.meta 2024-11-11T20:52:27,066 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T20:52:27,066 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:52:27,067 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T20:52:27,067 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T20:52:27,067 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T20:52:27,067 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T20:52:27,067 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:27,067 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T20:52:27,067 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T20:52:27,068 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-11T20:52:27,069 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T20:52:27,069 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:27,070 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/info/50c9486792674700b42a6281367b1d30, delay=946 ms 2024-11-11T20:52:27,072 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/info/50c9486792674700b42a6281367b1d30 2024-11-11T20:52:27,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:27,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-11T20:52:27,073 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T20:52:27,073 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:27,075 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/ns/14177cbb01e1428eae69f8fc7c41c980, delay=906 ms 2024-11-11T20:52:27,076 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T20:52:27,076 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,46645,1731358347011-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:27,076 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:27,076 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.Replication(171): 51ca66f7ee3c,46645,1731358347011 started 2024-11-11T20:52:27,076 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/ns/14177cbb01e1428eae69f8fc7c41c980 2024-11-11T20:52:27,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:27,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-11T20:52:27,078 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T20:52:27,078 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:27,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:27,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-11T20:52:27,079 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T20:52:27,079 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:27,081 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/table/5274cb5b4da043e981f865d5a5096a02, delay=920 ms 2024-11-11T20:52:27,082 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/table/5274cb5b4da043e981f865d5a5096a02 2024-11-11T20:52:27,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:27,083 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T20:52:27,083 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740 2024-11-11T20:52:27,084 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740 2024-11-11T20:52:27,084 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T20:52:27,084 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T20:52:27,085 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T20:52:27,086 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T20:52:27,087 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71050676, jitterRate=0.05873757600784302}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T20:52:27,087 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T20:52:27,089 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731358347067Writing region info on filesystem at 1731358347067Initializing all the Stores at 1731358347067Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358347067Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358347068 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358347068Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358347068Cleaning up temporary data from old regions at 1731358347084 (+16 ms)Running coprocessor post-open hooks at 1731358347087 (+3 ms)Region opened successfully at 1731358347089 (+2 ms) 2024-11-11T20:52:27,089 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731358347040 2024-11-11T20:52:27,092 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T20:52:27,093 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T20:52:27,093 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=51ca66f7ee3c,41961,1731358343700 2024-11-11T20:52:27,093 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:27,093 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.HRegionServer(1482): Serving as 51ca66f7ee3c,46645,1731358347011, RpcServer on 51ca66f7ee3c/172.17.0.2:46645, sessionid=0x100308f1e190003 2024-11-11T20:52:27,094 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,41961,1731358343700, state=OPEN 2024-11-11T20:52:27,094 DEBUG [RS:2;51ca66f7ee3c:46645 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T20:52:27,094 DEBUG [RS:2;51ca66f7ee3c:46645 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 51ca66f7ee3c,46645,1731358347011 2024-11-11T20:52:27,094 DEBUG [RS:2;51ca66f7ee3c:46645 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,46645,1731358347011' 2024-11-11T20:52:27,094 DEBUG [RS:2;51ca66f7ee3c:46645 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T20:52:27,094 DEBUG [RS:2;51ca66f7ee3c:46645 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T20:52:27,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46645-0x100308f1e190003, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:52:27,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:52:27,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:52:27,095 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,41961,1731358343700 2024-11-11T20:52:27,095 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:27,095 DEBUG [RS:2;51ca66f7ee3c:46645 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T20:52:27,095 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:27,095 DEBUG [RS:2;51ca66f7ee3c:46645 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T20:52:27,095 DEBUG [RS:2;51ca66f7ee3c:46645 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 51ca66f7ee3c,46645,1731358347011 2024-11-11T20:52:27,095 DEBUG [RS:2;51ca66f7ee3c:46645 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,46645,1731358347011' 2024-11-11T20:52:27,095 DEBUG [RS:2;51ca66f7ee3c:46645 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T20:52:27,096 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:27,096 DEBUG [RS:2;51ca66f7ee3c:46645 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T20:52:27,096 DEBUG [RS:2;51ca66f7ee3c:46645 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T20:52:27,096 INFO [RS:2;51ca66f7ee3c:46645 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T20:52:27,096 INFO [RS:2;51ca66f7ee3c:46645 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T20:52:27,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-11T20:52:27,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,41961,1731358343700 in 208 msec 2024-11-11T20:52:27,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T20:52:27,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 366 msec 2024-11-11T20:52:27,099 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 51ca66f7ee3c,34157,1731358343758 had 2 regions 2024-11-11T20:52:27,100 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 51ca66f7ee3c,34157,1731358343758, splitWal=true, meta=true, isMeta: false 2024-11-11T20:52:27,100 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,34157,1731358343758-splitting dir is empty, no logs to split. 2024-11-11T20:52:27,100 INFO [PEWorker-3 {}] master.SplitWALManager(105): 51ca66f7ee3c,34157,1731358343758 WAL count=0, meta=false 2024-11-11T20:52:27,101 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,34157,1731358343758-splitting dir is empty, no logs to split. 2024-11-11T20:52:27,101 INFO [PEWorker-3 {}] master.SplitWALManager(105): 51ca66f7ee3c,34157,1731358343758 WAL count=0, meta=false 2024-11-11T20:52:27,101 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 51ca66f7ee3c,34157,1731358343758 WAL splitting is done? wals=0, meta=false 2024-11-11T20:52:27,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731358344807, region=14831ea77f9af8d5ab1d6eaf567029e1, ASSIGN}] 2024-11-11T20:52:27,103 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731358344807, region=14831ea77f9af8d5ab1d6eaf567029e1, ASSIGN 2024-11-11T20:52:27,104 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731358344807, region=14831ea77f9af8d5ab1d6eaf567029e1, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T20:52:27,197 INFO [RS:2;51ca66f7ee3c:46645 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T20:52:27,198 INFO [RS:2;51ca66f7ee3c:46645 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=51ca66f7ee3c%2C46645%2C1731358347011, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,46645,1731358347011, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/oldWALs, maxLogs=55 2024-11-11T20:52:27,224 INFO [RS:2;51ca66f7ee3c:46645 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/WALs/51ca66f7ee3c,46645,1731358347011/51ca66f7ee3c%2C46645%2C1731358347011.1731358347200 2024-11-11T20:52:27,224 DEBUG [RS:2;51ca66f7ee3c:46645 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T20:52:27,255 DEBUG [51ca66f7ee3c:41949 {}] balancer.BalancerClusterState(204): Hosts are {51ca66f7ee3c=0} racks are {/default-rack=0} 2024-11-11T20:52:27,255 DEBUG [51ca66f7ee3c:41949 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T20:52:27,255 DEBUG [51ca66f7ee3c:41949 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T20:52:27,255 DEBUG [51ca66f7ee3c:41949 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T20:52:27,255 DEBUG [51ca66f7ee3c:41949 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T20:52:27,255 INFO [51ca66f7ee3c:41949 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T20:52:27,255 INFO [51ca66f7ee3c:41949 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T20:52:27,255 DEBUG [51ca66f7ee3c:41949 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T20:52:27,256 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=14831ea77f9af8d5ab1d6eaf567029e1, regionState=OPENING, regionLocation=51ca66f7ee3c,41961,1731358343700 2024-11-11T20:52:27,257 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 51ca66f7ee3c:34157 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 51ca66f7ee3c/172.17.0.2:34157 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:27,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,34157,1731358343758, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,34157,1731358343758, seqNum=-1, error=java.net.ConnectException: Call to address=51ca66f7ee3c:34157 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 51ca66f7ee3c/172.17.0.2:34157 2024-11-11T20:52:27,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,34157,1731358343758, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-11T20:52:27,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,34157,1731358343758, seqNum=-1 from cache 2024-11-11T20:52:27,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 51ca66f7ee3c:34157 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 51ca66f7ee3c/172.17.0.2:34157 2024-11-11T20:52:27,369 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:27,370 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,41961,1731358343700, seqNum=-1] 2024-11-11T20:52:27,370 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:27,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40105, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:27,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731358344807, region=14831ea77f9af8d5ab1d6eaf567029e1, ASSIGN because future has completed 2024-11-11T20:52:27,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 14831ea77f9af8d5ab1d6eaf567029e1, server=51ca66f7ee3c,41961,1731358343700}] 2024-11-11T20:52:27,535 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:27,535 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 14831ea77f9af8d5ab1d6eaf567029e1, NAME => 'testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:52:27,536 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731358344807 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:27,536 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:27,537 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:27,537 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:27,538 INFO [StoreOpener-14831ea77f9af8d5ab1d6eaf567029e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:27,539 INFO [StoreOpener-14831ea77f9af8d5ab1d6eaf567029e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14831ea77f9af8d5ab1d6eaf567029e1 columnFamilyName family 2024-11-11T20:52:27,539 DEBUG [StoreOpener-14831ea77f9af8d5ab1d6eaf567029e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:27,541 DEBUG [StoreFileOpener-14831ea77f9af8d5ab1d6eaf567029e1-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/family/782ec974e2cc4411b7057dfca5957718, delay=943 ms 2024-11-11T20:52:27,543 DEBUG [StoreOpener-14831ea77f9af8d5ab1d6eaf567029e1-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/family/782ec974e2cc4411b7057dfca5957718 2024-11-11T20:52:27,543 INFO [StoreOpener-14831ea77f9af8d5ab1d6eaf567029e1-1 {}] regionserver.HStore(327): Store=14831ea77f9af8d5ab1d6eaf567029e1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:27,543 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:27,544 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:27,545 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:27,545 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:27,545 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:27,547 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:27,547 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 14831ea77f9af8d5ab1d6eaf567029e1; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68262537, jitterRate=0.017191067337989807}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:52:27,547 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:27,548 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 14831ea77f9af8d5ab1d6eaf567029e1: Running coprocessor pre-open hook at 1731358347537Writing region info on filesystem at 1731358347537Initializing all the Stores at 1731358347537Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358347537Cleaning up temporary data from old regions at 1731358347545 (+8 ms)Running coprocessor post-open hooks at 1731358347547 (+2 ms)Region opened successfully at 1731358347548 (+1 ms) 2024-11-11T20:52:27,549 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1., pid=11, masterSystemTime=1731358347532 2024-11-11T20:52:27,552 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:27,552 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:27,553 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=14831ea77f9af8d5ab1d6eaf567029e1, regionState=OPEN, openSeqNum=10, regionLocation=51ca66f7ee3c,41961,1731358343700 2024-11-11T20:52:27,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 14831ea77f9af8d5ab1d6eaf567029e1, server=51ca66f7ee3c,41961,1731358343700 because future has completed 2024-11-11T20:52:27,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-11T20:52:27,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 14831ea77f9af8d5ab1d6eaf567029e1, server=51ca66f7ee3c,41961,1731358343700 in 177 msec 2024-11-11T20:52:27,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-11T20:52:27,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731358344807, region=14831ea77f9af8d5ab1d6eaf567029e1, ASSIGN in 456 msec 2024-11-11T20:52:27,560 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 51ca66f7ee3c,34157,1731358343758 after splitting done 2024-11-11T20:52:27,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 51ca66f7ee3c,34157,1731358343758, splitWal=true, meta=true in 841 msec 2024-11-11T20:52:27,594 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T20:52:27,594 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:52:27,594 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:52:27,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:27,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:27,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:52:27,595 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T20:52:27,595 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1345181180, stopped=false 2024-11-11T20:52:27,595 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=51ca66f7ee3c,41949,1731358343612 2024-11-11T20:52:27,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46645-0x100308f1e190003, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T20:52:27,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T20:52:27,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T20:52:27,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:27,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46645-0x100308f1e190003, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:27,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:27,597 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:52:27,598 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:52:27,598 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:52:27,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:27,599 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '51ca66f7ee3c,41961,1731358343700' ***** 2024-11-11T20:52:27,599 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T20:52:27,599 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '51ca66f7ee3c,46645,1731358347011' ***** 2024-11-11T20:52:27,599 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:52:27,599 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:52:27,599 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T20:52:27,599 INFO [RS:0;51ca66f7ee3c:41961 {}] bucket.BucketCache(1739): Disabling cache 2024-11-11T20:52:27,599 INFO [RS:2;51ca66f7ee3c:46645 {}] bucket.BucketCache(1739): Disabling cache 2024-11-11T20:52:27,599 INFO [RS:0;51ca66f7ee3c:41961 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/bucket.persistence 2024-11-11T20:52:27,599 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-11T20:52:27,599 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-11T20:52:27,599 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-11T20:52:27,600 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46645-0x100308f1e190003, quorum=127.0.0.1:60875, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:52:27,600 INFO [RS:2;51ca66f7ee3c:46645 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/720e2e21-9c98-fc6c-60e5-b8237f5c436f/bucket.persistence 2024-11-11T20:52:27,600 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-11T20:52:27,600 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-11T20:52:27,600 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-11T20:52:27,600 INFO [RS:0;51ca66f7ee3c:41961 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-11T20:52:27,600 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-11T20:52:27,601 INFO [RS:2;51ca66f7ee3c:46645 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-11T20:52:27,601 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-11T20:52:27,610 DEBUG [RS:0;51ca66f7ee3c:41961 {}] bucket.BucketCache(1488): Thread RS:0;51ca66f7ee3c:41961 started persisting bucket cache to file 2024-11-11T20:52:27,610 DEBUG [RS:0;51ca66f7ee3c:41961 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T20:52:27,611 DEBUG [RS:0;51ca66f7ee3c:41961 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T20:52:27,611 DEBUG [RS:2;51ca66f7ee3c:46645 {}] bucket.BucketCache(1488): Thread RS:2;51ca66f7ee3c:46645 started persisting bucket cache to file 2024-11-11T20:52:27,611 DEBUG [RS:2;51ca66f7ee3c:46645 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T20:52:27,611 DEBUG [RS:2;51ca66f7ee3c:46645 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T20:52:27,613 DEBUG [RS:0;51ca66f7ee3c:41961 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-11T20:52:27,613 DEBUG [RS:2;51ca66f7ee3c:46645 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-11T20:52:27,613 DEBUG [RS:0;51ca66f7ee3c:41961 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731358347610 2024-11-11T20:52:27,613 DEBUG [RS:2;51ca66f7ee3c:46645 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731358347611 2024-11-11T20:52:27,613 DEBUG [RS:0;51ca66f7ee3c:41961 {}] bucket.BucketCache(1510): Thread RS:0;51ca66f7ee3c:41961 finished persisting bucket cache to file, renaming 2024-11-11T20:52:27,613 DEBUG [RS:2;51ca66f7ee3c:46645 {}] bucket.BucketCache(1510): Thread RS:2;51ca66f7ee3c:46645 finished persisting bucket cache to file, renaming 2024-11-11T20:52:27,614 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T20:52:27,614 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T20:52:27,614 INFO [RS:0;51ca66f7ee3c:41961 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T20:52:27,614 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T20:52:27,614 INFO [RS:0;51ca66f7ee3c:41961 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T20:52:27,614 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T20:52:27,614 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer(3091): Received CLOSE for 14831ea77f9af8d5ab1d6eaf567029e1 2024-11-11T20:52:27,614 INFO [RS:2;51ca66f7ee3c:46645 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T20:52:27,614 INFO [RS:2;51ca66f7ee3c:46645 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T20:52:27,614 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.HRegionServer(959): stopping server 51ca66f7ee3c,46645,1731358347011 2024-11-11T20:52:27,614 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T20:52:27,614 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer(959): stopping server 51ca66f7ee3c,41961,1731358343700 2024-11-11T20:52:27,614 INFO [RS:2;51ca66f7ee3c:46645 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;51ca66f7ee3c:46645. 2024-11-11T20:52:27,614 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T20:52:27,614 INFO [RS:0;51ca66f7ee3c:41961 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;51ca66f7ee3c:41961. 2024-11-11T20:52:27,614 DEBUG [RS:2;51ca66f7ee3c:46645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:52:27,614 DEBUG [RS:2;51ca66f7ee3c:46645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:27,614 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-11T20:52:27,614 DEBUG [RS:0;51ca66f7ee3c:41961 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:52:27,614 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 14831ea77f9af8d5ab1d6eaf567029e1, disabling compactions & flushes 2024-11-11T20:52:27,615 DEBUG [RS:0;51ca66f7ee3c:41961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:27,615 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.HRegionServer(976): stopping server 51ca66f7ee3c,46645,1731358347011; all regions closed. 2024-11-11T20:52:27,615 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:27,615 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:27,615 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T20:52:27,615 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. after waiting 0 ms 2024-11-11T20:52:27,615 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T20:52:27,615 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:27,615 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T20:52:27,615 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T20:52:27,615 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-11T20:52:27,615 DEBUG [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 14831ea77f9af8d5ab1d6eaf567029e1=testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1.} 2024-11-11T20:52:27,615 DEBUG [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer(1351): Waiting on 14831ea77f9af8d5ab1d6eaf567029e1, 1588230740 2024-11-11T20:52:27,615 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-11T20:52:27,615 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T20:52:27,615 DEBUG [StoreFileCloser-14831ea77f9af8d5ab1d6eaf567029e1-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/family/782ec974e2cc4411b7057dfca5957718 2024-11-11T20:52:27,616 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T20:52:27,616 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T20:52:27,616 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T20:52:27,616 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T20:52:27,616 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-11T20:52:27,617 DEBUG [RS:2;51ca66f7ee3c:46645 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/oldWALs 2024-11-11T20:52:27,617 INFO [RS:2;51ca66f7ee3c:46645 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51ca66f7ee3c%2C46645%2C1731358347011:(num 1731358347200) 2024-11-11T20:52:27,617 DEBUG [RS:2;51ca66f7ee3c:46645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:27,617 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:52:27,617 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T20:52:27,618 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.ChoreService(370): Chore service for: regionserver/51ca66f7ee3c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T20:52:27,618 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T20:52:27,618 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T20:52:27,618 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T20:52:27,618 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T20:52:27,618 INFO [RS:2;51ca66f7ee3c:46645 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46645 2024-11-11T20:52:27,619 INFO [regionserver/51ca66f7ee3c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T20:52:27,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46645-0x100308f1e190003, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/51ca66f7ee3c,46645,1731358347011 2024-11-11T20:52:27,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T20:52:27,621 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T20:52:27,622 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/default/testBlockEvictionOnGracefulStop1731358344807/14831ea77f9af8d5ab1d6eaf567029e1/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-11T20:52:27,622 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [51ca66f7ee3c,46645,1731358347011] 2024-11-11T20:52:27,622 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:27,622 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 14831ea77f9af8d5ab1d6eaf567029e1: Waiting for close lock at 1731358347614Running coprocessor pre-close hooks at 1731358347614Disabling compacts and flushes for region at 1731358347614Disabling writes for close at 1731358347615 (+1 ms)Writing region close event to WAL at 1731358347616 (+1 ms)Running coprocessor post-close hooks at 1731358347622 (+6 ms)Closed at 1731358347622 2024-11-11T20:52:27,622 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1. 2024-11-11T20:52:27,623 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/51ca66f7ee3c,46645,1731358347011 already deleted, retry=false 2024-11-11T20:52:27,623 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 51ca66f7ee3c,46645,1731358347011 expired; onlineServers=1 2024-11-11T20:52:27,629 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dd8595b5ea1a4f01818481558a8ad008 2024-11-11T20:52:27,641 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/info/dd8595b5ea1a4f01818481558a8ad008 is 209, key is testBlockEvictionOnGracefulStop1731358344807,,1731358344812.14831ea77f9af8d5ab1d6eaf567029e1./info:regioninfo/1731358347552/Put/seqid=0 2024-11-11T20:52:27,642 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/info/dd8595b5ea1a4f01818481558a8ad008 2024-11-11T20:52:27,644 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/.tmp/info/dd8595b5ea1a4f01818481558a8ad008 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/info/dd8595b5ea1a4f01818481558a8ad008 2024-11-11T20:52:27,645 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/info/dd8595b5ea1a4f01818481558a8ad008, delay=942 ms 2024-11-11T20:52:27,646 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/info/dd8595b5ea1a4f01818481558a8ad008, entries=8, sequenceid=19, filesize=6.6 K 2024-11-11T20:52:27,646 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=19, compaction requested=false 2024-11-11T20:52:27,647 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/info/50c9486792674700b42a6281367b1d30 2024-11-11T20:52:27,647 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/info/dd8595b5ea1a4f01818481558a8ad008 2024-11-11T20:52:27,648 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/ns/14177cbb01e1428eae69f8fc7c41c980 2024-11-11T20:52:27,648 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/table/5274cb5b4da043e981f865d5a5096a02 2024-11-11T20:52:27,652 INFO [regionserver/51ca66f7ee3c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:52:27,652 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-11T20:52:27,652 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T20:52:27,653 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T20:52:27,653 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731358347615Running coprocessor pre-close hooks at 1731358347615Disabling compacts and flushes for region at 1731358347615Disabling writes for close at 1731358347616 (+1 ms)Obtaining lock to block concurrent updates at 1731358347616Preparing flush snapshotting stores in 1588230740 at 1731358347616Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731358347617 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731358347617Flushing 1588230740/info: creating writer at 1731358347617Flushing 1588230740/info: appending metadata at 1731358347641 (+24 ms)Flushing 1588230740/info: closing flushed file at 1731358347641Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@636726a2: reopening flushed file at 1731358347644 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=19, compaction requested=false at 1731358347646 (+2 ms)Writing region close event to WAL at 1731358347649 (+3 ms)Running coprocessor post-close hooks at 1731358347652 (+3 ms)Closed at 1731358347653 (+1 ms) 2024-11-11T20:52:27,653 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T20:52:27,659 INFO [regionserver/51ca66f7ee3c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:52:27,722 INFO [RS:2;51ca66f7ee3c:46645 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T20:52:27,723 INFO [RS:2;51ca66f7ee3c:46645 {}] regionserver.HRegionServer(1031): Exiting; stopping=51ca66f7ee3c,46645,1731358347011; zookeeper connection closed. 2024-11-11T20:52:27,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46645-0x100308f1e190003, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:52:27,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46645-0x100308f1e190003, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:52:27,723 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7e4380ef {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7e4380ef 2024-11-11T20:52:27,816 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer(976): stopping server 51ca66f7ee3c,41961,1731358343700; all regions closed. 2024-11-11T20:52:27,818 DEBUG [RS:0;51ca66f7ee3c:41961 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/oldWALs 2024-11-11T20:52:27,818 INFO [RS:0;51ca66f7ee3c:41961 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51ca66f7ee3c%2C41961%2C1731358343700.meta:.meta(num 1731358347046) 2024-11-11T20:52:27,820 DEBUG [RS:0;51ca66f7ee3c:41961 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/oldWALs 2024-11-11T20:52:27,820 INFO [RS:0;51ca66f7ee3c:41961 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51ca66f7ee3c%2C41961%2C1731358343700:(num 1731358344202) 2024-11-11T20:52:27,820 DEBUG [RS:0;51ca66f7ee3c:41961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:27,820 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:52:27,820 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T20:52:27,820 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.ChoreService(370): Chore service for: regionserver/51ca66f7ee3c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T20:52:27,821 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T20:52:27,821 INFO [regionserver/51ca66f7ee3c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T20:52:27,821 INFO [RS:0;51ca66f7ee3c:41961 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41961 2024-11-11T20:52:27,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/51ca66f7ee3c,41961,1731358343700 2024-11-11T20:52:27,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T20:52:27,823 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T20:52:27,824 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [51ca66f7ee3c,41961,1731358343700] 2024-11-11T20:52:27,825 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/51ca66f7ee3c,41961,1731358343700 already deleted, retry=false 2024-11-11T20:52:27,825 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 51ca66f7ee3c,41961,1731358343700 expired; onlineServers=0 2024-11-11T20:52:27,825 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '51ca66f7ee3c,41949,1731358343612' ***** 2024-11-11T20:52:27,825 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T20:52:27,825 INFO [M:0;51ca66f7ee3c:41949 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T20:52:27,825 INFO [M:0;51ca66f7ee3c:41949 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T20:52:27,825 DEBUG [M:0;51ca66f7ee3c:41949 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T20:52:27,825 DEBUG [M:0;51ca66f7ee3c:41949 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T20:52:27,825 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T20:52:27,825 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.large.0-1731358343972 {}] cleaner.HFileCleaner(306): Exit Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.large.0-1731358343972,5,FailOnTimeoutGroup] 2024-11-11T20:52:27,825 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.small.0-1731358343972 {}] cleaner.HFileCleaner(306): Exit Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.small.0-1731358343972,5,FailOnTimeoutGroup] 2024-11-11T20:52:27,825 INFO [M:0;51ca66f7ee3c:41949 {}] hbase.ChoreService(370): Chore service for: master/51ca66f7ee3c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T20:52:27,826 INFO [M:0;51ca66f7ee3c:41949 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T20:52:27,826 DEBUG [M:0;51ca66f7ee3c:41949 {}] master.HMaster(1795): Stopping service threads 2024-11-11T20:52:27,826 INFO [M:0;51ca66f7ee3c:41949 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T20:52:27,826 INFO [M:0;51ca66f7ee3c:41949 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:52:27,826 INFO [M:0;51ca66f7ee3c:41949 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:52:27,826 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T20:52:27,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T20:52:27,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:27,827 DEBUG [M:0;51ca66f7ee3c:41949 {}] zookeeper.ZKUtil(347): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T20:52:27,827 WARN [M:0;51ca66f7ee3c:41949 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T20:52:27,827 INFO [M:0;51ca66f7ee3c:41949 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/.lastflushedseqids 2024-11-11T20:52:27,833 INFO [M:0;51ca66f7ee3c:41949 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:52:27,834 INFO [M:0;51ca66f7ee3c:41949 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T20:52:27,834 DEBUG [M:0;51ca66f7ee3c:41949 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T20:52:27,834 INFO [M:0;51ca66f7ee3c:41949 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:27,834 DEBUG [M:0;51ca66f7ee3c:41949 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:27,834 DEBUG [M:0;51ca66f7ee3c:41949 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T20:52:27,834 DEBUG [M:0;51ca66f7ee3c:41949 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:27,834 INFO [M:0;51ca66f7ee3c:41949 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-11T20:52:27,849 DEBUG [M:0;51ca66f7ee3c:41949 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 71f7670cea2a4968b35192d23a95429d 2024-11-11T20:52:27,862 DEBUG [M:0;51ca66f7ee3c:41949 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/71f7670cea2a4968b35192d23a95429d is 82, key is hbase:meta,,1/info:regioninfo/1731358347092/Put/seqid=0 2024-11-11T20:52:27,863 INFO [M:0;51ca66f7ee3c:41949 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/71f7670cea2a4968b35192d23a95429d 2024-11-11T20:52:27,870 DEBUG [M:0;51ca66f7ee3c:41949 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 66482515ca5349128500ec48441b5221 2024-11-11T20:52:27,884 DEBUG [M:0;51ca66f7ee3c:41949 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/66482515ca5349128500ec48441b5221 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731358345201/Put/seqid=0 2024-11-11T20:52:27,884 INFO [M:0;51ca66f7ee3c:41949 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/66482515ca5349128500ec48441b5221 2024-11-11T20:52:27,892 DEBUG [M:0;51ca66f7ee3c:41949 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e7de18ffd3f64e0d96de9c43a4d594f5 2024-11-11T20:52:27,906 DEBUG [M:0;51ca66f7ee3c:41949 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e7de18ffd3f64e0d96de9c43a4d594f5 is 69, key is 51ca66f7ee3c,41961,1731358343700/rs:state/1731358344015/Put/seqid=0 2024-11-11T20:52:27,907 INFO [M:0;51ca66f7ee3c:41949 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e7de18ffd3f64e0d96de9c43a4d594f5 2024-11-11T20:52:27,909 INFO [M:0;51ca66f7ee3c:41949 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e7de18ffd3f64e0d96de9c43a4d594f5 2024-11-11T20:52:27,909 DEBUG [M:0;51ca66f7ee3c:41949 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/71f7670cea2a4968b35192d23a95429d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/71f7670cea2a4968b35192d23a95429d 2024-11-11T20:52:27,911 INFO [M:0;51ca66f7ee3c:41949 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/71f7670cea2a4968b35192d23a95429d, entries=14, sequenceid=102, filesize=5.9 K 2024-11-11T20:52:27,912 DEBUG [M:0;51ca66f7ee3c:41949 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/66482515ca5349128500ec48441b5221 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/66482515ca5349128500ec48441b5221 2024-11-11T20:52:27,913 INFO [M:0;51ca66f7ee3c:41949 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/66482515ca5349128500ec48441b5221, entries=11, sequenceid=102, filesize=6.5 K 2024-11-11T20:52:27,914 DEBUG [M:0;51ca66f7ee3c:41949 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e7de18ffd3f64e0d96de9c43a4d594f5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e7de18ffd3f64e0d96de9c43a4d594f5 2024-11-11T20:52:27,915 INFO [M:0;51ca66f7ee3c:41949 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e7de18ffd3f64e0d96de9c43a4d594f5 2024-11-11T20:52:27,915 INFO [M:0;51ca66f7ee3c:41949 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bf2a6991-cb7f-9402-b0ee-5ca03eb35a2f/7519108c-9c29-3fa4-8570-b72e638914e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e7de18ffd3f64e0d96de9c43a4d594f5, entries=3, sequenceid=102, filesize=5.3 K 2024-11-11T20:52:27,916 INFO [M:0;51ca66f7ee3c:41949 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=102, compaction requested=false 2024-11-11T20:52:27,921 INFO [M:0;51ca66f7ee3c:41949 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:27,922 DEBUG [M:0;51ca66f7ee3c:41949 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731358347834Disabling compacts and flushes for region at 1731358347834Disabling writes for close at 1731358347834Obtaining lock to block concurrent updates at 1731358347834Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731358347834Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1731358347835 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731358347835Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731358347835Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731358347862 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731358347862Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731358347865 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731358347883 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731358347883Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731358347886 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731358347905 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731358347905Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60cb3399: reopening flushed file at 1731358347909 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fdc9dc2: reopening flushed file at 1731358347911 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50fa1037: reopening flushed file at 1731358347913 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=102, compaction requested=false at 1731358347916 (+3 ms)Writing region close event to WAL at 1731358347921 (+5 ms)Closed at 1731358347921 2024-11-11T20:52:27,923 INFO [M:0;51ca66f7ee3c:41949 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T20:52:27,923 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T20:52:27,923 INFO [M:0;51ca66f7ee3c:41949 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41949 2024-11-11T20:52:27,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:52:27,924 INFO [RS:0;51ca66f7ee3c:41961 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T20:52:27,924 INFO [M:0;51ca66f7ee3c:41949 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T20:52:27,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100308f1e190001, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:52:27,924 INFO [RS:0;51ca66f7ee3c:41961 {}] regionserver.HRegionServer(1031): Exiting; stopping=51ca66f7ee3c,41961,1731358343700; zookeeper connection closed. 2024-11-11T20:52:27,924 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5a469fd5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5a469fd5 2024-11-11T20:52:27,925 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-11T20:52:28,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:52:28,026 INFO [M:0;51ca66f7ee3c:41949 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T20:52:28,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41949-0x100308f1e190000, quorum=127.0.0.1:60875, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:52:28,028 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T20:52:28,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-11T20:52:28,030 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:28,036 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 50) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=221 (was 240), ProcessCount=11 (was 11), AvailableMemoryMB=5633 (was 5763)