2024-12-08 04:01:39,596 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-08 04:01:39,613 main DEBUG Took 0.015105 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-08 04:01:39,613 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-08 04:01:39,613 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-08 04:01:39,614 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-08 04:01:39,615 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 04:01:39,621 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-08 04:01:39,631 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 04:01:39,632 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 04:01:39,633 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 04:01:39,633 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 04:01:39,634 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 04:01:39,634 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 04:01:39,634 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 04:01:39,635 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 04:01:39,635 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 04:01:39,635 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 04:01:39,636 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 04:01:39,636 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 04:01:39,637 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 04:01:39,637 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 04:01:39,637 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 04:01:39,637 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 04:01:39,638 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 04:01:39,638 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 04:01:39,638 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 04:01:39,639 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 04:01:39,639 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 04:01:39,639 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 04:01:39,639 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 04:01:39,640 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 04:01:39,640 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 04:01:39,640 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-08 04:01:39,641 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 04:01:39,643 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-08 04:01:39,644 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-08 04:01:39,645 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-08 04:01:39,646 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-08 04:01:39,646 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-08 04:01:39,653 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-08 04:01:39,655 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-08 04:01:39,657 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-08 04:01:39,657 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-08 04:01:39,658 main DEBUG createAppenders(={Console}) 2024-12-08 04:01:39,658 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-08 04:01:39,658 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-08 04:01:39,659 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-08 04:01:39,659 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-08 04:01:39,659 main DEBUG OutputStream closed 2024-12-08 04:01:39,660 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-08 04:01:39,660 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-08 04:01:39,660 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-08 04:01:39,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-08 04:01:39,725 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-08 04:01:39,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-08 04:01:39,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-08 04:01:39,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-08 04:01:39,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-08 04:01:39,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-08 04:01:39,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-08 04:01:39,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-08 04:01:39,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-08 04:01:39,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-08 04:01:39,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-08 04:01:39,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-08 04:01:39,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-08 04:01:39,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-08 04:01:39,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-08 04:01:39,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-08 04:01:39,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-08 04:01:39,734 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08 04:01:39,734 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-08 04:01:39,734 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-08 04:01:39,735 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-08T04:01:39,746 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-08 04:01:39,748 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-08 04:01:39,749 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08T04:01:39,961 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2 2024-12-08T04:01:39,984 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=372, ProcessCount=11, AvailableMemoryMB=17886 2024-12-08T04:01:40,079 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-08T04:01:40,187 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:40,204 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/cluster_b485c177-607e-7c3e-1773-492883f677f3, deleteOnExit=true 2024-12-08T04:01:40,349 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/cluster_b485c177-607e-7c3e-1773-492883f677f3/zookeeper_0, clientPort=60499, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/cluster_b485c177-607e-7c3e-1773-492883f677f3/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/cluster_b485c177-607e-7c3e-1773-492883f677f3/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-08T04:01:40,361 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60499 2024-12-08T04:01:40,362 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:40,438 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68 with version=8 2024-12-08T04:01:40,438 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/hbase-staging 2024-12-08T04:01:40,518 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-08T04:01:40,733 INFO [Time-limited test {}] client.ConnectionUtils(128): master/32df4b4bb439:0 server-side Connection retries=45 2024-12-08T04:01:40,741 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:40,743 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:40,753 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T04:01:40,753 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:40,753 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T04:01:40,883 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-08T04:01:40,962 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T04:01:41,000 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-08T04:01:41,009 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-08T04:01:41,013 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T04:01:41,044 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 110614 (auto-detected) 2024-12-08T04:01:41,045 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-08T04:01:41,063 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35353 2024-12-08T04:01:41,084 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35353 connecting to ZooKeeper ensemble=127.0.0.1:60499 2024-12-08T04:01:41,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:353530x0, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T04:01:41,203 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35353-0x10003a3dc860000 connected 2024-12-08T04:01:41,300 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:41,301 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:41,309 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T04:01:41,313 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68, hbase.cluster.distributed=false 2024-12-08T04:01:41,334 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T04:01:41,339 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35353 2024-12-08T04:01:41,339 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35353 2024-12-08T04:01:41,340 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35353 2024-12-08T04:01:41,340 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35353 2024-12-08T04:01:41,341 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35353 2024-12-08T04:01:41,408 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/32df4b4bb439:0 server-side Connection retries=45 2024-12-08T04:01:41,409 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:41,410 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:41,410 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T04:01:41,410 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:41,410 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T04:01:41,413 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T04:01:41,415 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T04:01:41,415 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37281 2024-12-08T04:01:41,417 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37281 connecting to ZooKeeper ensemble=127.0.0.1:60499 2024-12-08T04:01:41,417 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:41,418 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:41,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:372810x0, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T04:01:41,438 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:372810x0, quorum=127.0.0.1:60499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T04:01:41,439 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37281-0x10003a3dc860001 connected 2024-12-08T04:01:41,446 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T04:01:41,457 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/bucket.cache 2024-12-08T04:01:41,462 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-08T04:01:41,463 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-08T04:01:41,465 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-08T04:01:41,466 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-08T04:01:41,466 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-08T04:01:41,473 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-08T04:01:41,474 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-08T04:01:41,474 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/bucket.persistence is complete. 2024-12-08T04:01:41,480 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T04:01:41,484 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T04:01:41,491 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T04:01:41,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37281 2024-12-08T04:01:41,496 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37281 2024-12-08T04:01:41,496 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37281 2024-12-08T04:01:41,497 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37281 2024-12-08T04:01:41,498 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37281 2024-12-08T04:01:41,516 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/32df4b4bb439:0 server-side Connection retries=45 2024-12-08T04:01:41,516 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:41,517 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:41,517 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T04:01:41,517 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:41,517 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T04:01:41,518 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T04:01:41,518 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T04:01:41,519 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39845 2024-12-08T04:01:41,521 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39845 connecting to ZooKeeper ensemble=127.0.0.1:60499 2024-12-08T04:01:41,522 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:41,523 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:41,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:398450x0, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T04:01:41,547 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:398450x0, quorum=127.0.0.1:60499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T04:01:41,547 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39845-0x10003a3dc860002 connected 2024-12-08T04:01:41,548 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T04:01:41,550 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/bucket.cache 2024-12-08T04:01:41,550 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-08T04:01:41,550 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-08T04:01:41,551 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-08T04:01:41,551 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-08T04:01:41,551 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-08T04:01:41,551 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-08T04:01:41,552 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-08T04:01:41,552 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/bucket.persistence is complete. 2024-12-08T04:01:41,552 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T04:01:41,554 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T04:01:41,557 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T04:01:41,559 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39845 2024-12-08T04:01:41,559 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39845 2024-12-08T04:01:41,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39845 2024-12-08T04:01:41,567 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39845 2024-12-08T04:01:41,567 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39845 2024-12-08T04:01:41,582 DEBUG [M:0;32df4b4bb439:35353 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;32df4b4bb439:35353 2024-12-08T04:01:41,583 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/32df4b4bb439,35353,1733630500581 2024-12-08T04:01:41,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T04:01:41,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T04:01:41,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T04:01:41,600 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/32df4b4bb439,35353,1733630500581 2024-12-08T04:01:41,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T04:01:41,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:41,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T04:01:41,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:41,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:41,630 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T04:01:41,631 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/32df4b4bb439,35353,1733630500581 from backup master directory 2024-12-08T04:01:41,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T04:01:41,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/32df4b4bb439,35353,1733630500581 2024-12-08T04:01:41,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T04:01:41,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T04:01:41,644 WARN [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T04:01:41,644 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=32df4b4bb439,35353,1733630500581 2024-12-08T04:01:41,646 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-08T04:01:41,648 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-08T04:01:41,655 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/hbase.id] with ID: 243c03b7-8234-4aeb-abf2-479a6b3454b2 2024-12-08T04:01:41,656 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/.tmp/hbase.id 2024-12-08T04:01:41,661 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/hbase.id] 2024-12-08T04:01:41,696 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:41,700 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-08T04:01:41,717 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-08T04:01:41,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:41,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:41,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:41,766 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T04:01:41,768 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-08T04:01:41,809 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:41,814 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T04:01:41,866 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store 2024-12-08T04:01:41,886 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-08T04:01:41,889 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:41,891 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T04:01:41,891 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:41,892 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:41,894 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T04:01:41,894 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:41,895 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:41,895 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733630501891Disabling compacts and flushes for region at 1733630501891Disabling writes for close at 1733630501894 (+3 ms)Writing region close event to WAL at 1733630501894Closed at 1733630501894 2024-12-08T04:01:41,896 WARN [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/.initializing 2024-12-08T04:01:41,896 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/WALs/32df4b4bb439,35353,1733630500581 2024-12-08T04:01:41,910 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T04:01:41,923 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32df4b4bb439%2C35353%2C1733630500581, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/WALs/32df4b4bb439,35353,1733630500581, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/oldWALs, maxLogs=10 2024-12-08T04:01:41,963 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/WALs/32df4b4bb439,35353,1733630500581/32df4b4bb439%2C35353%2C1733630500581.1733630501929 2024-12-08T04:01:41,964 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T04:01:41,965 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-08T04:01:41,965 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:41,969 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:41,970 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:42,003 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:42,028 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-08T04:01:42,032 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:42,033 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:42,034 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:42,038 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-08T04:01:42,038 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:42,039 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:42,039 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:42,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-08T04:01:42,043 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:42,044 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:42,044 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:42,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-08T04:01:42,049 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:42,049 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:42,050 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:42,053 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:42,054 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:42,057 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:42,057 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:42,060 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T04:01:42,062 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:42,068 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T04:01:42,070 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64992770, jitterRate=-0.03153225779533386}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T04:01:42,077 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733630501982Initializing all the Stores at 1733630501982Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630501982Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630501983 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630501984 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630501984Cleaning up temporary data from old regions at 1733630502057 (+73 ms)Region opened successfully at 1733630502077 (+20 ms) 2024-12-08T04:01:42,079 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-08T04:01:42,110 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb47360, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32df4b4bb439/172.17.0.2:0 2024-12-08T04:01:42,143 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-08T04:01:42,155 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-08T04:01:42,155 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-08T04:01:42,158 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-08T04:01:42,158 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-08T04:01:42,164 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-08T04:01:42,164 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-08T04:01:42,187 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-08T04:01:42,196 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-08T04:01:42,226 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-08T04:01:42,228 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-08T04:01:42,230 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-08T04:01:42,237 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-08T04:01:42,240 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-08T04:01:42,245 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-08T04:01:42,254 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-08T04:01:42,256 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-08T04:01:42,267 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-08T04:01:42,286 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-08T04:01:42,295 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-08T04:01:42,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T04:01:42,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T04:01:42,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T04:01:42,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:42,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:42,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:42,322 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=32df4b4bb439,35353,1733630500581, sessionid=0x10003a3dc860000, setting cluster-up flag (Was=false) 2024-12-08T04:01:42,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:42,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:42,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:42,379 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-08T04:01:42,382 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32df4b4bb439,35353,1733630500581 2024-12-08T04:01:42,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:42,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:42,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:42,612 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-08T04:01:42,615 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32df4b4bb439,35353,1733630500581 2024-12-08T04:01:42,621 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-08T04:01:42,672 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.HRegionServer(746): ClusterId : 243c03b7-8234-4aeb-abf2-479a6b3454b2 2024-12-08T04:01:42,672 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer(746): ClusterId : 243c03b7-8234-4aeb-abf2-479a6b3454b2 2024-12-08T04:01:42,675 DEBUG [RS:1;32df4b4bb439:39845 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T04:01:42,675 DEBUG [RS:0;32df4b4bb439:37281 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T04:01:42,686 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-08T04:01:42,695 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-08T04:01:42,701 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-08T04:01:42,706 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 32df4b4bb439,35353,1733630500581 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-08T04:01:42,728 DEBUG [RS:0;32df4b4bb439:37281 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T04:01:42,728 DEBUG [RS:1;32df4b4bb439:39845 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T04:01:42,728 DEBUG [RS:0;32df4b4bb439:37281 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T04:01:42,728 DEBUG [RS:1;32df4b4bb439:39845 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T04:01:42,729 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/32df4b4bb439:0, corePoolSize=5, maxPoolSize=5 2024-12-08T04:01:42,729 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/32df4b4bb439:0, corePoolSize=5, maxPoolSize=5 2024-12-08T04:01:42,729 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/32df4b4bb439:0, corePoolSize=5, maxPoolSize=5 2024-12-08T04:01:42,729 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/32df4b4bb439:0, corePoolSize=5, maxPoolSize=5 2024-12-08T04:01:42,730 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/32df4b4bb439:0, corePoolSize=10, maxPoolSize=10 2024-12-08T04:01:42,730 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,730 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/32df4b4bb439:0, corePoolSize=2, maxPoolSize=2 2024-12-08T04:01:42,730 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,732 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733630532732 2024-12-08T04:01:42,734 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-08T04:01:42,735 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-08T04:01:42,735 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T04:01:42,736 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-08T04:01:42,739 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-08T04:01:42,740 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-08T04:01:42,740 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:42,740 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-08T04:01:42,741 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-08T04:01:42,740 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-08T04:01:42,742 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,744 DEBUG [RS:0;32df4b4bb439:37281 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T04:01:42,744 DEBUG [RS:1;32df4b4bb439:39845 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T04:01:42,745 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-08T04:01:42,745 DEBUG [RS:1;32df4b4bb439:39845 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64bde709, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32df4b4bb439/172.17.0.2:0 2024-12-08T04:01:42,745 DEBUG [RS:0;32df4b4bb439:37281 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79ef2186, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32df4b4bb439/172.17.0.2:0 2024-12-08T04:01:42,746 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-08T04:01:42,747 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-08T04:01:42,749 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-08T04:01:42,750 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-08T04:01:42,751 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.large.0-1733630502751,5,FailOnTimeoutGroup] 2024-12-08T04:01:42,752 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.small.0-1733630502752,5,FailOnTimeoutGroup] 2024-12-08T04:01:42,752 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,753 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-08T04:01:42,754 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,754 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,756 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-08T04:01:42,756 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68 2024-12-08T04:01:42,765 DEBUG [RS:0;32df4b4bb439:37281 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;32df4b4bb439:37281 2024-12-08T04:01:42,765 DEBUG [RS:1;32df4b4bb439:39845 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;32df4b4bb439:39845 2024-12-08T04:01:42,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:42,768 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T04:01:42,768 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T04:01:42,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-08T04:01:42,768 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T04:01:42,768 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T04:01:42,768 DEBUG [RS:1;32df4b4bb439:39845 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T04:01:42,768 DEBUG [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T04:01:42,771 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.HRegionServer(2659): reportForDuty to master=32df4b4bb439,35353,1733630500581 with port=39845, startcode=1733630501515 2024-12-08T04:01:42,771 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer(2659): reportForDuty to master=32df4b4bb439,35353,1733630500581 with port=37281, startcode=1733630501379 2024-12-08T04:01:42,772 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T04:01:42,772 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:42,772 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:42,773 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-08T04:01:42,776 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T04:01:42,777 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:42,777 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:42,777 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-08T04:01:42,781 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T04:01:42,781 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:42,781 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:42,781 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-08T04:01:42,784 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T04:01:42,785 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:42,785 DEBUG [RS:0;32df4b4bb439:37281 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T04:01:42,785 DEBUG [RS:1;32df4b4bb439:39845 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T04:01:42,785 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:42,785 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T04:01:42,786 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740 2024-12-08T04:01:42,786 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740 2024-12-08T04:01:42,787 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T04:01:42,787 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T04:01:42,788 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T04:01:42,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T04:01:42,794 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T04:01:42,795 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71995268, jitterRate=0.07281309366226196}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T04:01:42,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733630502767Initializing all the Stores at 1733630502767Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630502767Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630502768 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630502768Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630502768Cleaning up temporary data from old regions at 1733630502787 (+19 ms)Region opened successfully at 1733630502799 (+12 ms) 2024-12-08T04:01:42,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T04:01:42,800 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T04:01:42,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T04:01:42,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T04:01:42,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T04:01:42,801 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T04:01:42,802 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733630502799Disabling compacts and flushes for region at 1733630502799Disabling writes for close at 1733630502800 (+1 ms)Writing region close event to WAL at 1733630502801 (+1 ms)Closed at 1733630502801 2024-12-08T04:01:42,805 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T04:01:42,805 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-08T04:01:42,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T04:01:42,821 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T04:01:42,823 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-08T04:01:42,850 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41879, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T04:01:42,850 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46077, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T04:01:42,858 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35353 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 32df4b4bb439,37281,1733630501379 2024-12-08T04:01:42,861 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35353 {}] master.ServerManager(517): Registering regionserver=32df4b4bb439,37281,1733630501379 2024-12-08T04:01:42,872 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35353 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 32df4b4bb439,39845,1733630501515 2024-12-08T04:01:42,872 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35353 {}] master.ServerManager(517): Registering regionserver=32df4b4bb439,39845,1733630501515 2024-12-08T04:01:42,875 DEBUG [RS:1;32df4b4bb439:39845 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68 2024-12-08T04:01:42,875 DEBUG [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68 2024-12-08T04:01:42,875 DEBUG [RS:1;32df4b4bb439:39845 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-08T04:01:42,875 DEBUG [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-08T04:01:42,875 DEBUG [RS:1;32df4b4bb439:39845 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T04:01:42,875 DEBUG [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T04:01:42,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T04:01:42,918 DEBUG [RS:0;32df4b4bb439:37281 {}] zookeeper.ZKUtil(111): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/32df4b4bb439,37281,1733630501379 2024-12-08T04:01:42,918 DEBUG [RS:1;32df4b4bb439:39845 {}] zookeeper.ZKUtil(111): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/32df4b4bb439,39845,1733630501515 2024-12-08T04:01:42,918 WARN [RS:0;32df4b4bb439:37281 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T04:01:42,918 WARN [RS:1;32df4b4bb439:39845 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T04:01:42,919 INFO [RS:0;32df4b4bb439:37281 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T04:01:42,919 INFO [RS:1;32df4b4bb439:39845 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T04:01:42,919 DEBUG [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,37281,1733630501379 2024-12-08T04:01:42,919 DEBUG [RS:1;32df4b4bb439:39845 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,39845,1733630501515 2024-12-08T04:01:42,921 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [32df4b4bb439,39845,1733630501515] 2024-12-08T04:01:42,921 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [32df4b4bb439,37281,1733630501379] 2024-12-08T04:01:42,944 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T04:01:42,944 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T04:01:42,956 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T04:01:42,956 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T04:01:42,961 INFO [RS:0;32df4b4bb439:37281 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T04:01:42,961 INFO [RS:1;32df4b4bb439:39845 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T04:01:42,961 INFO [RS:0;32df4b4bb439:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,961 INFO [RS:1;32df4b4bb439:39845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,962 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T04:01:42,962 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T04:01:42,967 INFO [RS:1;32df4b4bb439:39845 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T04:01:42,967 INFO [RS:0;32df4b4bb439:37281 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T04:01:42,968 INFO [RS:1;32df4b4bb439:39845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,968 INFO [RS:0;32df4b4bb439:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,969 DEBUG [RS:0;32df4b4bb439:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,969 DEBUG [RS:1;32df4b4bb439:39845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,969 DEBUG [RS:0;32df4b4bb439:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,969 DEBUG [RS:1;32df4b4bb439:39845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,969 DEBUG [RS:0;32df4b4bb439:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,969 DEBUG [RS:1;32df4b4bb439:39845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,969 DEBUG [RS:0;32df4b4bb439:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,969 DEBUG [RS:1;32df4b4bb439:39845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,969 DEBUG [RS:0;32df4b4bb439:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,969 DEBUG [RS:1;32df4b4bb439:39845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,969 DEBUG [RS:0;32df4b4bb439:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/32df4b4bb439:0, corePoolSize=2, maxPoolSize=2 2024-12-08T04:01:42,969 DEBUG [RS:1;32df4b4bb439:39845 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/32df4b4bb439:0, corePoolSize=2, maxPoolSize=2 2024-12-08T04:01:42,969 DEBUG [RS:0;32df4b4bb439:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,969 DEBUG [RS:1;32df4b4bb439:39845 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,969 DEBUG [RS:0;32df4b4bb439:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,969 DEBUG [RS:1;32df4b4bb439:39845 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,970 DEBUG [RS:0;32df4b4bb439:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,970 DEBUG [RS:1;32df4b4bb439:39845 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,970 DEBUG [RS:0;32df4b4bb439:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,970 DEBUG [RS:1;32df4b4bb439:39845 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,970 DEBUG [RS:0;32df4b4bb439:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,970 DEBUG [RS:1;32df4b4bb439:39845 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,970 DEBUG [RS:0;32df4b4bb439:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,970 DEBUG [RS:1;32df4b4bb439:39845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:42,970 DEBUG [RS:0;32df4b4bb439:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/32df4b4bb439:0, corePoolSize=3, maxPoolSize=3 2024-12-08T04:01:42,970 DEBUG [RS:1;32df4b4bb439:39845 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/32df4b4bb439:0, corePoolSize=3, maxPoolSize=3 2024-12-08T04:01:42,970 DEBUG [RS:0;32df4b4bb439:37281 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0, corePoolSize=3, maxPoolSize=3 2024-12-08T04:01:42,970 DEBUG [RS:1;32df4b4bb439:39845 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0, corePoolSize=3, maxPoolSize=3 2024-12-08T04:01:42,971 INFO [RS:1;32df4b4bb439:39845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,971 INFO [RS:0;32df4b4bb439:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,971 INFO [RS:1;32df4b4bb439:39845 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,972 INFO [RS:1;32df4b4bb439:39845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,972 INFO [RS:0;32df4b4bb439:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,972 INFO [RS:1;32df4b4bb439:39845 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,972 INFO [RS:0;32df4b4bb439:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,972 INFO [RS:1;32df4b4bb439:39845 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,972 INFO [RS:1;32df4b4bb439:39845 {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,39845,1733630501515-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T04:01:42,972 INFO [RS:0;32df4b4bb439:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,972 INFO [RS:0;32df4b4bb439:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,972 INFO [RS:0;32df4b4bb439:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,37281,1733630501379-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T04:01:42,974 WARN [32df4b4bb439:35353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T04:01:42,997 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T04:01:42,997 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T04:01:42,999 INFO [RS:0;32df4b4bb439:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,37281,1733630501379-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,999 INFO [RS:1;32df4b4bb439:39845 {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,39845,1733630501515-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,999 INFO [RS:1;32df4b4bb439:39845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,999 INFO [RS:0;32df4b4bb439:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:42,999 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.Replication(171): 32df4b4bb439,39845,1733630501515 started 2024-12-08T04:01:42,999 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.Replication(171): 32df4b4bb439,37281,1733630501379 started 2024-12-08T04:01:43,022 INFO [RS:0;32df4b4bb439:37281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:43,022 INFO [RS:1;32df4b4bb439:39845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:43,022 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer(1482): Serving as 32df4b4bb439,37281,1733630501379, RpcServer on 32df4b4bb439/172.17.0.2:37281, sessionid=0x10003a3dc860001 2024-12-08T04:01:43,022 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.HRegionServer(1482): Serving as 32df4b4bb439,39845,1733630501515, RpcServer on 32df4b4bb439/172.17.0.2:39845, sessionid=0x10003a3dc860002 2024-12-08T04:01:43,023 DEBUG [RS:1;32df4b4bb439:39845 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T04:01:43,023 DEBUG [RS:0;32df4b4bb439:37281 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T04:01:43,023 DEBUG [RS:0;32df4b4bb439:37281 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 32df4b4bb439,37281,1733630501379 2024-12-08T04:01:43,023 DEBUG [RS:1;32df4b4bb439:39845 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 32df4b4bb439,39845,1733630501515 2024-12-08T04:01:43,023 DEBUG [RS:1;32df4b4bb439:39845 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32df4b4bb439,39845,1733630501515' 2024-12-08T04:01:43,023 DEBUG [RS:0;32df4b4bb439:37281 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32df4b4bb439,37281,1733630501379' 2024-12-08T04:01:43,023 DEBUG [RS:0;32df4b4bb439:37281 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T04:01:43,023 DEBUG [RS:1;32df4b4bb439:39845 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T04:01:43,024 DEBUG [RS:1;32df4b4bb439:39845 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T04:01:43,024 DEBUG [RS:0;32df4b4bb439:37281 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T04:01:43,025 DEBUG [RS:1;32df4b4bb439:39845 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T04:01:43,025 DEBUG [RS:0;32df4b4bb439:37281 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T04:01:43,025 DEBUG [RS:1;32df4b4bb439:39845 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T04:01:43,025 DEBUG [RS:0;32df4b4bb439:37281 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T04:01:43,025 DEBUG [RS:1;32df4b4bb439:39845 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 32df4b4bb439,39845,1733630501515 2024-12-08T04:01:43,025 DEBUG [RS:0;32df4b4bb439:37281 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 32df4b4bb439,37281,1733630501379 2024-12-08T04:01:43,025 DEBUG [RS:1;32df4b4bb439:39845 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32df4b4bb439,39845,1733630501515' 2024-12-08T04:01:43,025 DEBUG [RS:0;32df4b4bb439:37281 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32df4b4bb439,37281,1733630501379' 2024-12-08T04:01:43,025 DEBUG [RS:1;32df4b4bb439:39845 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T04:01:43,025 DEBUG [RS:0;32df4b4bb439:37281 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T04:01:43,026 DEBUG [RS:1;32df4b4bb439:39845 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T04:01:43,026 DEBUG [RS:0;32df4b4bb439:37281 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T04:01:43,026 DEBUG [RS:1;32df4b4bb439:39845 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T04:01:43,026 DEBUG [RS:0;32df4b4bb439:37281 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T04:01:43,027 INFO [RS:1;32df4b4bb439:39845 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T04:01:43,027 INFO [RS:0;32df4b4bb439:37281 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T04:01:43,027 INFO [RS:0;32df4b4bb439:37281 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T04:01:43,027 INFO [RS:1;32df4b4bb439:39845 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T04:01:43,133 INFO [RS:0;32df4b4bb439:37281 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T04:01:43,133 INFO [RS:1;32df4b4bb439:39845 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T04:01:43,134 INFO [RS:1;32df4b4bb439:39845 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=32df4b4bb439%2C39845%2C1733630501515, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,39845,1733630501515, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/oldWALs, maxLogs=55 2024-12-08T04:01:43,134 INFO [RS:0;32df4b4bb439:37281 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=32df4b4bb439%2C37281%2C1733630501379, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,37281,1733630501379, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/oldWALs, maxLogs=55 2024-12-08T04:01:43,156 INFO [RS:1;32df4b4bb439:39845 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,39845,1733630501515/32df4b4bb439%2C39845%2C1733630501515.1733630503141 2024-12-08T04:01:43,156 INFO [RS:0;32df4b4bb439:37281 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,37281,1733630501379/32df4b4bb439%2C37281%2C1733630501379.1733630503141 2024-12-08T04:01:43,156 DEBUG [RS:1;32df4b4bb439:39845 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T04:01:43,156 DEBUG [RS:0;32df4b4bb439:37281 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T04:01:43,227 DEBUG [32df4b4bb439:35353 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-08T04:01:43,240 DEBUG [32df4b4bb439:35353 {}] balancer.BalancerClusterState(204): Hosts are {32df4b4bb439=0} racks are {/default-rack=0} 2024-12-08T04:01:43,242 DEBUG [32df4b4bb439:35353 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T04:01:43,242 DEBUG [32df4b4bb439:35353 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T04:01:43,243 DEBUG [32df4b4bb439:35353 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T04:01:43,243 DEBUG [32df4b4bb439:35353 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T04:01:43,243 INFO [32df4b4bb439:35353 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T04:01:43,243 INFO [32df4b4bb439:35353 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T04:01:43,243 DEBUG [32df4b4bb439:35353 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T04:01:43,249 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=32df4b4bb439,37281,1733630501379 2024-12-08T04:01:43,253 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32df4b4bb439,37281,1733630501379, state=OPENING 2024-12-08T04:01:43,279 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-08T04:01:43,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:43,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:43,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:43,289 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:43,289 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:43,289 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:43,291 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T04:01:43,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=32df4b4bb439,37281,1733630501379}] 2024-12-08T04:01:43,470 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T04:01:43,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53831, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T04:01:43,479 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T04:01:43,480 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T04:01:43,481 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T04:01:43,482 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=32df4b4bb439%2C37281%2C1733630501379.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,37281,1733630501379, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/oldWALs, maxLogs=55 2024-12-08T04:01:43,501 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,37281,1733630501379/32df4b4bb439%2C37281%2C1733630501379.meta.1733630503485.meta 2024-12-08T04:01:43,501 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T04:01:43,502 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T04:01:43,503 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T04:01:43,505 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T04:01:43,509 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T04:01:43,512 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T04:01:43,513 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:43,513 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T04:01:43,513 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T04:01:43,515 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-08T04:01:43,516 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T04:01:43,516 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:43,516 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:43,516 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-08T04:01:43,517 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T04:01:43,517 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:43,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:43,517 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-08T04:01:43,518 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T04:01:43,518 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:43,518 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:43,518 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-08T04:01:43,519 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T04:01:43,519 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:43,519 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:43,519 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T04:01:43,521 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740 2024-12-08T04:01:43,521 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740 2024-12-08T04:01:43,522 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T04:01:43,522 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T04:01:43,522 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T04:01:43,523 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T04:01:43,524 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70544295, jitterRate=0.05119191110134125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T04:01:43,524 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T04:01:43,526 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733630503514Writing region info on filesystem at 1733630503514Initializing all the Stores at 1733630503514Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630503514Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630503515 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630503515Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630503515Cleaning up temporary data from old regions at 1733630503522 (+7 ms)Running coprocessor post-open hooks at 1733630503524 (+2 ms)Region opened successfully at 1733630503525 (+1 ms) 2024-12-08T04:01:43,530 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733630503461 2024-12-08T04:01:43,539 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T04:01:43,540 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T04:01:43,540 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,37281,1733630501379 2024-12-08T04:01:43,542 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32df4b4bb439,37281,1733630501379, state=OPEN 2024-12-08T04:01:43,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T04:01:43,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T04:01:43,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T04:01:43,567 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:43,568 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:43,568 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:43,568 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=32df4b4bb439,37281,1733630501379 2024-12-08T04:01:43,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-08T04:01:43,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=32df4b4bb439,37281,1733630501379 in 276 msec 2024-12-08T04:01:43,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-08T04:01:43,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 761 msec 2024-12-08T04:01:43,579 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T04:01:43,579 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-08T04:01:43,601 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:43,602 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,37281,1733630501379, seqNum=-1] 2024-12-08T04:01:43,624 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:43,627 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40457, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:43,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 994 msec 2024-12-08T04:01:43,645 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733630503645, completionTime=-1 2024-12-08T04:01:43,648 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-08T04:01:43,649 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-08T04:01:43,677 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-08T04:01:43,678 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733630563678 2024-12-08T04:01:43,678 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733630623678 2024-12-08T04:01:43,678 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-12-08T04:01:43,681 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-08T04:01:43,687 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,35353,1733630500581-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:43,687 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,35353,1733630500581-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:43,687 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,35353,1733630500581-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:43,689 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-32df4b4bb439:35353, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:43,689 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:43,690 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:43,697 DEBUG [master/32df4b4bb439:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-08T04:01:43,718 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.074sec 2024-12-08T04:01:43,719 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-08T04:01:43,721 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-08T04:01:43,721 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-08T04:01:43,722 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-08T04:01:43,722 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-08T04:01:43,734 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,35353,1733630500581-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T04:01:43,735 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,35353,1733630500581-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-08T04:01:43,736 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-08T04:01:43,737 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-08T04:01:43,737 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,35353,1733630500581-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:43,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:43,783 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-08T04:01:43,783 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-08T04:01:43,786 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,35353,-1 for getting cluster id 2024-12-08T04:01:43,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:01:43,795 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '243c03b7-8234-4aeb-abf2-479a6b3454b2' 2024-12-08T04:01:43,797 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:01:43,798 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "243c03b7-8234-4aeb-abf2-479a6b3454b2" 2024-12-08T04:01:43,798 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc75c2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:43,798 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,35353,-1] 2024-12-08T04:01:43,802 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:01:43,804 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:43,804 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:01:43,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:43,807 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:43,813 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,37281,1733630501379, seqNum=-1] 2024-12-08T04:01:43,813 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:43,816 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:43,832 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=32df4b4bb439,35353,1733630500581 2024-12-08T04:01:43,836 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T04:01:43,840 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 32df4b4bb439,35353,1733630500581 2024-12-08T04:01:43,841 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3bd01ab2 2024-12-08T04:01:43,842 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T04:01:43,844 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T04:01:43,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35353 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733630503833', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T04:01:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35353 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733630503833 2024-12-08T04:01:43,860 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733630503833 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T04:01:43,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35353 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733630503833" procId is: 4 2024-12-08T04:01:43,862 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:43,864 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733630503833 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T04:01:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T04:01:43,875 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733630503833-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6e1d9f840481246ce467be1827839ea0, NAME => 'testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733630503833', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68 2024-12-08T04:01:43,881 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733630503833-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:43,881 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733630503833-pool-0 {}] regionserver.HRegion(1722): Closing 6e1d9f840481246ce467be1827839ea0, disabling compactions & flushes 2024-12-08T04:01:43,881 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733630503833-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:43,881 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733630503833-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:43,881 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733630503833-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. after waiting 0 ms 2024-12-08T04:01:43,881 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733630503833-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:43,882 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733630503833-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:43,882 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733630503833-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6e1d9f840481246ce467be1827839ea0: Waiting for close lock at 1733630503881Disabling compacts and flushes for region at 1733630503881Disabling writes for close at 1733630503881Writing region close event to WAL at 1733630503882 (+1 ms)Closed at 1733630503882 2024-12-08T04:01:43,884 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733630503833 execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T04:01:43,889 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733630503884"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733630503884"}]},"ts":"1733630503884"} 2024-12-08T04:01:43,894 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T04:01:43,895 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733630503833 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T04:01:43,898 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733630503833","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630503895"}]},"ts":"1733630503895"} 2024-12-08T04:01:43,902 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733630503833, state=ENABLING in hbase:meta 2024-12-08T04:01:43,903 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(204): Hosts are {32df4b4bb439=0} racks are {/default-rack=0} 2024-12-08T04:01:43,904 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T04:01:43,904 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T04:01:43,904 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T04:01:43,904 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T04:01:43,904 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T04:01:43,904 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T04:01:43,904 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T04:01:43,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733630503833, region=6e1d9f840481246ce467be1827839ea0, ASSIGN}] 2024-12-08T04:01:43,908 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733630503833, region=6e1d9f840481246ce467be1827839ea0, ASSIGN 2024-12-08T04:01:43,909 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733630503833, region=6e1d9f840481246ce467be1827839ea0, ASSIGN; state=OFFLINE, location=32df4b4bb439,39845,1733630501515; forceNewPlan=false, retain=false 2024-12-08T04:01:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T04:01:44,066 INFO [32df4b4bb439:35353 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-08T04:01:44,067 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6e1d9f840481246ce467be1827839ea0, regionState=OPENING, regionLocation=32df4b4bb439,39845,1733630501515 2024-12-08T04:01:44,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733630503833, region=6e1d9f840481246ce467be1827839ea0, ASSIGN because future has completed 2024-12-08T04:01:44,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e1d9f840481246ce467be1827839ea0, server=32df4b4bb439,39845,1733630501515}] 2024-12-08T04:01:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T04:01:44,226 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T04:01:44,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33789, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T04:01:44,230 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:44,230 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e1d9f840481246ce467be1827839ea0, NAME => 'testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0.', STARTKEY => '', ENDKEY => ''} 2024-12-08T04:01:44,230 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733630503833 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:44,230 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:44,231 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:44,231 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:44,231 INFO [StoreOpener-6e1d9f840481246ce467be1827839ea0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:44,235 INFO [StoreOpener-6e1d9f840481246ce467be1827839ea0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e1d9f840481246ce467be1827839ea0 columnFamilyName family 2024-12-08T04:01:44,235 DEBUG [StoreOpener-6e1d9f840481246ce467be1827839ea0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:44,235 INFO [StoreOpener-6e1d9f840481246ce467be1827839ea0-1 {}] regionserver.HStore(327): Store=6e1d9f840481246ce467be1827839ea0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:44,235 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:44,236 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:44,236 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:44,236 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:44,236 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:44,237 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:44,242 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T04:01:44,243 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6e1d9f840481246ce467be1827839ea0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73358311, jitterRate=0.09312401711940765}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T04:01:44,243 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:44,244 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6e1d9f840481246ce467be1827839ea0: Running coprocessor pre-open hook at 1733630504231Writing region info on filesystem at 1733630504231Initializing all the Stores at 1733630504231Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630504231Cleaning up temporary data from old regions at 1733630504237 (+6 ms)Running coprocessor post-open hooks at 1733630504243 (+6 ms)Region opened successfully at 1733630504244 (+1 ms) 2024-12-08T04:01:44,245 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0., pid=6, masterSystemTime=1733630504225 2024-12-08T04:01:44,249 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:44,249 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:44,249 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6e1d9f840481246ce467be1827839ea0, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,39845,1733630501515 2024-12-08T04:01:44,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e1d9f840481246ce467be1827839ea0, server=32df4b4bb439,39845,1733630501515 because future has completed 2024-12-08T04:01:44,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-08T04:01:44,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6e1d9f840481246ce467be1827839ea0, server=32df4b4bb439,39845,1733630501515 in 183 msec 2024-12-08T04:01:44,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-08T04:01:44,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733630503833, region=6e1d9f840481246ce467be1827839ea0, ASSIGN in 351 msec 2024-12-08T04:01:44,261 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733630503833 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T04:01:44,261 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733630503833","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630504261"}]},"ts":"1733630504261"} 2024-12-08T04:01:44,263 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733630503833, state=ENABLED in hbase:meta 2024-12-08T04:01:44,265 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733630503833 execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T04:01:44,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733630503833 in 411 msec 2024-12-08T04:01:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T04:01:44,504 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733630503833 get assigned. Timeout = 60000ms 2024-12-08T04:01:44,504 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733630503833 completed 2024-12-08T04:01:44,508 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T04:01:44,514 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733630503833 assigned to meta. Checking AM states. 2024-12-08T04:01:44,515 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T04:01:44,516 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733630503833 assigned. 2024-12-08T04:01:44,526 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733630503833', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0., hostname=32df4b4bb439,39845,1733630501515, seqNum=2] 2024-12-08T04:01:44,527 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:44,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:44,539 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6e1d9f840481246ce467be1827839ea0 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-08T04:01:44,562 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 62f548e7bc7444239bf062cfb720a613 2024-12-08T04:01:44,593 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/.tmp/family/62f548e7bc7444239bf062cfb720a613 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-08T04:01:44,597 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/.tmp/family/62f548e7bc7444239bf062cfb720a613 2024-12-08T04:01:44,626 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/.tmp/family/62f548e7bc7444239bf062cfb720a613 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/family/62f548e7bc7444239bf062cfb720a613 2024-12-08T04:01:44,628 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/family/62f548e7bc7444239bf062cfb720a613, delay=914 ms 2024-12-08T04:01:44,629 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/family/62f548e7bc7444239bf062cfb720a613, entries=2, sequenceid=6, filesize=4.9 K 2024-12-08T04:01:44,635 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 6e1d9f840481246ce467be1827839ea0 in 95ms, sequenceid=6, compaction requested=false 2024-12-08T04:01:44,636 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733630503833' 2024-12-08T04:01:44,637 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6e1d9f840481246ce467be1827839ea0: 2024-12-08T04:01:45,547 DEBUG [hfile-prefetch-1733630504628 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 62f548e7bc7444239bf062cfb720a613_0. 2024-12-08T04:01:45,553 DEBUG [hfile-prefetch-1733630504628 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 62f548e7bc7444239bf062cfb720a613_117. 2024-12-08T04:01:45,553 DEBUG [hfile-prefetch-1733630504628 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/family/62f548e7bc7444239bf062cfb720a613, with total blocks 2, and data blocks 1 2024-12-08T04:01:45,553 DEBUG [hfile-prefetch-1733630504628 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-08T04:01:45,554 DEBUG [hfile-prefetch-1733630504628 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/family/62f548e7bc7444239bf062cfb720a613, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-08T04:01:45,554 DEBUG [hfile-prefetch-1733630504628 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/family/62f548e7bc7444239bf062cfb720a613. Will sleep 100ms and try the verification again. 2024-12-08T04:01:45,652 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35353 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=6e1d9f840481246ce467be1827839ea0, source=32df4b4bb439,39845,1733630501515, destination=32df4b4bb439,37281,1733630501379, warming up region on 32df4b4bb439,37281,1733630501379 2024-12-08T04:01:45,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35353 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=6e1d9f840481246ce467be1827839ea0, source=32df4b4bb439,39845,1733630501515, destination=32df4b4bb439,37281,1733630501379, running balancer 2024-12-08T04:01:45,654 DEBUG [hfile-prefetch-1733630504628 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/family/62f548e7bc7444239bf062cfb720a613, with total blocks 2, and data blocks 1 2024-12-08T04:01:45,654 DEBUG [hfile-prefetch-1733630504628 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-12-08T04:01:45,654 DEBUG [hfile-prefetch-1733630504628 {}] bucket.BucketCache(2293): found block for file 62f548e7bc7444239bf062cfb720a613 in the backing map. Acquiring read lock for offset 0 2024-12-08T04:01:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35353 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733630503833, region=6e1d9f840481246ce467be1827839ea0, REOPEN/MOVE 2024-12-08T04:01:45,654 DEBUG [hfile-prefetch-1733630504628 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/family/62f548e7bc7444239bf062cfb720a613 has now been fully cached. 2024-12-08T04:01:45,655 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733630503833, region=6e1d9f840481246ce467be1827839ea0, REOPEN/MOVE 2024-12-08T04:01:45,655 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-08T04:01:45,655 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-08T04:01:45,655 DEBUG [hfile-prefetch-1733630504628 {}] hfile.PrefetchExecutor(121): Prefetch completed for 62f548e7bc7444239bf062cfb720a613 2024-12-08T04:01:45,655 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37281 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:45,655 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T04:01:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37281 {}] regionserver.HRegion(7855): Warmup {ENCODED => 6e1d9f840481246ce467be1827839ea0, NAME => 'testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0.', STARTKEY => '', ENDKEY => ''} 2024-12-08T04:01:45,655 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-12-08T04:01:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37281 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:45,656 INFO [StoreOpener-6e1d9f840481246ce467be1827839ea0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:45,657 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6e1d9f840481246ce467be1827839ea0, regionState=CLOSING, regionLocation=32df4b4bb439,39845,1733630501515 2024-12-08T04:01:45,657 INFO [StoreOpener-6e1d9f840481246ce467be1827839ea0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e1d9f840481246ce467be1827839ea0 columnFamilyName family 2024-12-08T04:01:45,657 DEBUG [StoreOpener-6e1d9f840481246ce467be1827839ea0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:45,660 DEBUG [StoreOpener-6e1d9f840481246ce467be1827839ea0-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/family/62f548e7bc7444239bf062cfb720a613 2024-12-08T04:01:45,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733630503833, region=6e1d9f840481246ce467be1827839ea0, REOPEN/MOVE because future has completed 2024-12-08T04:01:45,660 INFO [StoreOpener-6e1d9f840481246ce467be1827839ea0-1 {}] regionserver.HStore(327): Store=6e1d9f840481246ce467be1827839ea0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37281 {}] regionserver.HRegion(1722): Closing 6e1d9f840481246ce467be1827839ea0, disabling compactions & flushes 2024-12-08T04:01:45,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37281 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37281 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37281 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. after waiting 0 ms 2024-12-08T04:01:45,661 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-08T04:01:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37281 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:45,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6e1d9f840481246ce467be1827839ea0, server=32df4b4bb439,39845,1733630501515}] 2024-12-08T04:01:45,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37281 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37281 {}] regionserver.HRegion(1676): Region close journal for 6e1d9f840481246ce467be1827839ea0: Waiting for close lock at 1733630505661Disabling compacts and flushes for region at 1733630505661Disabling writes for close at 1733630505661Writing region close event to WAL at 1733630505662 (+1 ms)Closed at 1733630505662 2024-12-08T04:01:45,669 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-08T04:01:45,669 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-08T04:01:45,669 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T04:01:45,669 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-08T04:01:45,672 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-08T04:01:45,672 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733630505669 2024-12-08T04:01:45,672 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-08T04:01:45,674 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-12-08T04:01:45,674 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1733630505655 2024-12-08T04:01:45,674 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-08T04:01:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35353 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-08T04:01:45,823 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:45,823 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-08T04:01:45,824 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 6e1d9f840481246ce467be1827839ea0, disabling compactions & flushes 2024-12-08T04:01:45,824 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:45,824 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:45,824 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. after waiting 0 ms 2024-12-08T04:01:45,825 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:45,827 DEBUG [StoreFileCloser-6e1d9f840481246ce467be1827839ea0-family-1 {}] bucket.BucketCache(768): Removing file 62f548e7bc7444239bf062cfb720a613 for region 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:45,827 DEBUG [StoreFileCloser-6e1d9f840481246ce467be1827839ea0-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 62f548e7bc7444239bf062cfb720a613, starting offset: 0, end offset: 9223372036854775807 2024-12-08T04:01:45,829 DEBUG [StoreFileCloser-6e1d9f840481246ce467be1827839ea0-family-1 {}] bucket.BucketCache(856): removed key 62f548e7bc7444239bf062cfb720a613_0 from back map with offset lock 2094080 in the evict process 2024-12-08T04:01:45,834 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-08T04:01:45,835 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:45,835 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 6e1d9f840481246ce467be1827839ea0: Waiting for close lock at 1733630505824Running coprocessor pre-close hooks at 1733630505824Disabling compacts and flushes for region at 1733630505824Disabling writes for close at 1733630505824Writing region close event to WAL at 1733630505829 (+5 ms)Running coprocessor post-close hooks at 1733630505834 (+5 ms)Closed at 1733630505835 (+1 ms) 2024-12-08T04:01:45,836 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 6e1d9f840481246ce467be1827839ea0 move to 32df4b4bb439,37281,1733630501379 record at close sequenceid=6 2024-12-08T04:01:45,839 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:45,840 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6e1d9f840481246ce467be1827839ea0, regionState=CLOSED 2024-12-08T04:01:45,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6e1d9f840481246ce467be1827839ea0, server=32df4b4bb439,39845,1733630501515 because future has completed 2024-12-08T04:01:45,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-08T04:01:45,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 6e1d9f840481246ce467be1827839ea0, server=32df4b4bb439,39845,1733630501515 in 183 msec 2024-12-08T04:01:45,847 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733630503833, region=6e1d9f840481246ce467be1827839ea0, REOPEN/MOVE; state=CLOSED, location=32df4b4bb439,37281,1733630501379; forceNewPlan=false, retain=false 2024-12-08T04:01:45,872 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-08T04:01:45,872 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-08T04:01:45,872 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T04:01:45,872 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-08T04:01:45,876 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-08T04:01:45,876 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733630505872 2024-12-08T04:01:45,876 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-08T04:01:45,997 INFO [32df4b4bb439:35353 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-08T04:01:45,998 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6e1d9f840481246ce467be1827839ea0, regionState=OPENING, regionLocation=32df4b4bb439,37281,1733630501379 2024-12-08T04:01:46,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733630503833, region=6e1d9f840481246ce467be1827839ea0, REOPEN/MOVE because future has completed 2024-12-08T04:01:46,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e1d9f840481246ce467be1827839ea0, server=32df4b4bb439,37281,1733630501379}] 2024-12-08T04:01:46,161 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:46,162 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e1d9f840481246ce467be1827839ea0, NAME => 'testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0.', STARTKEY => '', ENDKEY => ''} 2024-12-08T04:01:46,163 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733630503833 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:46,163 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:46,163 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:46,163 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:46,165 INFO [StoreOpener-6e1d9f840481246ce467be1827839ea0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:46,167 INFO [StoreOpener-6e1d9f840481246ce467be1827839ea0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e1d9f840481246ce467be1827839ea0 columnFamilyName family 2024-12-08T04:01:46,167 DEBUG [StoreOpener-6e1d9f840481246ce467be1827839ea0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:46,169 DEBUG [StoreFileOpener-6e1d9f840481246ce467be1827839ea0-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/family/62f548e7bc7444239bf062cfb720a613, delay=956 ms 2024-12-08T04:01:46,170 DEBUG [StoreOpener-6e1d9f840481246ce467be1827839ea0-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/family/62f548e7bc7444239bf062cfb720a613 2024-12-08T04:01:46,170 INFO [StoreOpener-6e1d9f840481246ce467be1827839ea0-1 {}] regionserver.HStore(327): Store=6e1d9f840481246ce467be1827839ea0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:46,170 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:46,171 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:46,172 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:46,172 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:46,172 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:46,176 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:46,177 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 6e1d9f840481246ce467be1827839ea0; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68555267, jitterRate=0.02155308425426483}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T04:01:46,177 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:46,177 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 6e1d9f840481246ce467be1827839ea0: Running coprocessor pre-open hook at 1733630506164Writing region info on filesystem at 1733630506164Initializing all the Stores at 1733630506164Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630506164Cleaning up temporary data from old regions at 1733630506172 (+8 ms)Running coprocessor post-open hooks at 1733630506177 (+5 ms)Region opened successfully at 1733630506177 2024-12-08T04:01:46,178 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0., pid=9, masterSystemTime=1733630506156 2024-12-08T04:01:46,181 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:46,181 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:46,181 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6e1d9f840481246ce467be1827839ea0, regionState=OPEN, openSeqNum=10, regionLocation=32df4b4bb439,37281,1733630501379 2024-12-08T04:01:46,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e1d9f840481246ce467be1827839ea0, server=32df4b4bb439,37281,1733630501379 because future has completed 2024-12-08T04:01:46,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-08T04:01:46,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 6e1d9f840481246ce467be1827839ea0, server=32df4b4bb439,37281,1733630501379 in 183 msec 2024-12-08T04:01:46,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733630503833, region=6e1d9f840481246ce467be1827839ea0, REOPEN/MOVE in 535 msec 2024-12-08T04:01:46,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-08T04:01:46,208 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T04:01:46,208 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T04:01:46,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:46,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:46,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:46,213 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-08T04:01:46,213 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-12-08T04:01:46,213 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=32df4b4bb439,35353,1733630500581 2024-12-08T04:01:46,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T04:01:46,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T04:01:46,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T04:01:46,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:46,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:46,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:46,251 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T04:01:46,251 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T04:01:46,252 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T04:01:46,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:46,252 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T04:01:46,253 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T04:01:46,253 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T04:01:46,253 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '32df4b4bb439,37281,1733630501379' ***** 2024-12-08T04:01:46,254 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T04:01:46,254 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '32df4b4bb439,39845,1733630501515' ***** 2024-12-08T04:01:46,254 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T04:01:46,254 INFO [RS:0;32df4b4bb439:37281 {}] bucket.BucketCache(1738): Disabling cache 2024-12-08T04:01:46,255 INFO [RS:1;32df4b4bb439:39845 {}] bucket.BucketCache(1738): Disabling cache 2024-12-08T04:01:46,255 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-08T04:01:46,255 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-08T04:01:46,255 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-08T04:01:46,255 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-08T04:01:46,255 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-08T04:01:46,255 INFO [RS:0;32df4b4bb439:37281 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/bucket.persistence 2024-12-08T04:01:46,255 INFO [RS:0;32df4b4bb439:37281 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-08T04:01:46,256 INFO [RS:1;32df4b4bb439:39845 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/bucket.persistence 2024-12-08T04:01:46,256 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-08T04:01:46,256 INFO [RS:1;32df4b4bb439:39845 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-08T04:01:46,256 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-08T04:01:46,256 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-08T04:01:46,266 DEBUG [RS:0;32df4b4bb439:37281 {}] bucket.BucketCache(1487): Thread RS:0;32df4b4bb439:37281 started persisting bucket cache to file 2024-12-08T04:01:46,266 DEBUG [RS:0;32df4b4bb439:37281 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T04:01:46,266 DEBUG [RS:0;32df4b4bb439:37281 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-08T04:01:46,266 DEBUG [RS:1;32df4b4bb439:39845 {}] bucket.BucketCache(1487): Thread RS:1;32df4b4bb439:39845 started persisting bucket cache to file 2024-12-08T04:01:46,266 DEBUG [RS:1;32df4b4bb439:39845 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T04:01:46,266 DEBUG [RS:1;32df4b4bb439:39845 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-08T04:01:46,269 DEBUG [RS:0;32df4b4bb439:37281 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-08T04:01:46,269 DEBUG [RS:1;32df4b4bb439:39845 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-08T04:01:46,269 DEBUG [RS:0;32df4b4bb439:37281 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733630506266 2024-12-08T04:01:46,269 DEBUG [RS:1;32df4b4bb439:39845 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733630506266 2024-12-08T04:01:46,269 DEBUG [RS:1;32df4b4bb439:39845 {}] bucket.BucketCache(1509): Thread RS:1;32df4b4bb439:39845 finished persisting bucket cache to file, renaming 2024-12-08T04:01:46,269 DEBUG [RS:0;32df4b4bb439:37281 {}] bucket.BucketCache(1509): Thread RS:0;32df4b4bb439:37281 finished persisting bucket cache to file, renaming 2024-12-08T04:01:46,269 WARN [RS:0;32df4b4bb439:37281 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-08T04:01:46,269 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T04:01:46,269 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T04:01:46,269 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T04:01:46,270 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T04:01:46,270 INFO [RS:0;32df4b4bb439:37281 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T04:01:46,270 INFO [RS:1;32df4b4bb439:39845 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T04:01:46,270 INFO [RS:0;32df4b4bb439:37281 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T04:01:46,270 INFO [RS:1;32df4b4bb439:39845 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T04:01:46,270 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.HRegionServer(959): stopping server 32df4b4bb439,39845,1733630501515 2024-12-08T04:01:46,270 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer(3091): Received CLOSE for 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:46,270 INFO [RS:1;32df4b4bb439:39845 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T04:01:46,270 INFO [RS:1;32df4b4bb439:39845 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;32df4b4bb439:39845. 2024-12-08T04:01:46,270 DEBUG [RS:1;32df4b4bb439:39845 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T04:01:46,270 DEBUG [RS:1;32df4b4bb439:39845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:46,271 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer(959): stopping server 32df4b4bb439,37281,1733630501379 2024-12-08T04:01:46,271 INFO [RS:0;32df4b4bb439:37281 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T04:01:46,271 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.HRegionServer(976): stopping server 32df4b4bb439,39845,1733630501515; all regions closed. 2024-12-08T04:01:46,271 INFO [RS:0;32df4b4bb439:37281 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;32df4b4bb439:37281. 2024-12-08T04:01:46,271 DEBUG [RS:0;32df4b4bb439:37281 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T04:01:46,271 DEBUG [RS:0;32df4b4bb439:37281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:46,271 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T04:01:46,271 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T04:01:46,271 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T04:01:46,271 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T04:01:46,271 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-08T04:01:46,272 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6e1d9f840481246ce467be1827839ea0, disabling compactions & flushes 2024-12-08T04:01:46,272 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-08T04:01:46,272 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:46,272 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:46,272 DEBUG [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 6e1d9f840481246ce467be1827839ea0=testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0.} 2024-12-08T04:01:46,272 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-08T04:01:46,272 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. after waiting 0 ms 2024-12-08T04:01:46,272 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:46,272 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T04:01:46,272 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T04:01:46,273 DEBUG [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6e1d9f840481246ce467be1827839ea0 2024-12-08T04:01:46,273 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T04:01:46,273 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T04:01:46,273 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T04:01:46,273 DEBUG [StoreFileCloser-6e1d9f840481246ce467be1827839ea0-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/family/62f548e7bc7444239bf062cfb720a613 2024-12-08T04:01:46,273 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-08T04:01:46,275 INFO [regionserver/32df4b4bb439:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T04:01:46,275 INFO [regionserver/32df4b4bb439:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T04:01:46,277 DEBUG [RS:1;32df4b4bb439:39845 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/oldWALs 2024-12-08T04:01:46,277 INFO [RS:1;32df4b4bb439:39845 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32df4b4bb439%2C39845%2C1733630501515:(num 1733630503141) 2024-12-08T04:01:46,277 DEBUG [RS:1;32df4b4bb439:39845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:46,277 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T04:01:46,277 INFO [RS:1;32df4b4bb439:39845 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T04:01:46,277 INFO [RS:1;32df4b4bb439:39845 {}] hbase.ChoreService(370): Chore service for: regionserver/32df4b4bb439:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-08T04:01:46,278 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T04:01:46,278 INFO [regionserver/32df4b4bb439:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T04:01:46,278 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T04:01:46,278 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T04:01:46,278 INFO [RS:1;32df4b4bb439:39845 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T04:01:46,278 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnRegionMove1733630503833/6e1d9f840481246ce467be1827839ea0/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-08T04:01:46,278 INFO [RS:1;32df4b4bb439:39845 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39845 2024-12-08T04:01:46,279 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:46,279 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6e1d9f840481246ce467be1827839ea0: Waiting for close lock at 1733630506271Running coprocessor pre-close hooks at 1733630506272 (+1 ms)Disabling compacts and flushes for region at 1733630506272Disabling writes for close at 1733630506272Writing region close event to WAL at 1733630506274 (+2 ms)Running coprocessor post-close hooks at 1733630506279 (+5 ms)Closed at 1733630506279 2024-12-08T04:01:46,279 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0. 2024-12-08T04:01:46,283 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ee08f55c0146486cbb84b1f1ced16afd 2024-12-08T04:01:46,300 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/info/ee08f55c0146486cbb84b1f1ced16afd is 205, key is testBlockEvictionOnRegionMove1733630503833,,1733630503846.6e1d9f840481246ce467be1827839ea0./info:regioninfo/1733630506181/Put/seqid=0 2024-12-08T04:01:46,301 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/info/ee08f55c0146486cbb84b1f1ced16afd 2024-12-08T04:01:46,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/32df4b4bb439,39845,1733630501515 2024-12-08T04:01:46,303 INFO [RS:1;32df4b4bb439:39845 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T04:01:46,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T04:01:46,310 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fffd289f20954e5da06692674b90a0fd 2024-12-08T04:01:46,312 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [32df4b4bb439,39845,1733630501515] 2024-12-08T04:01:46,320 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/32df4b4bb439,39845,1733630501515 already deleted, retry=false 2024-12-08T04:01:46,320 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 32df4b4bb439,39845,1733630501515 expired; onlineServers=1 2024-12-08T04:01:46,324 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/ns/fffd289f20954e5da06692674b90a0fd is 43, key is default/ns:d/1733630503631/Put/seqid=0 2024-12-08T04:01:46,325 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/ns/fffd289f20954e5da06692674b90a0fd 2024-12-08T04:01:46,333 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9bf1843f6b7a4162ae69ae15b13f5cf1 2024-12-08T04:01:46,344 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/table/9bf1843f6b7a4162ae69ae15b13f5cf1 is 78, key is testBlockEvictionOnRegionMove1733630503833/table:state/1733630504261/Put/seqid=0 2024-12-08T04:01:46,344 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/table/9bf1843f6b7a4162ae69ae15b13f5cf1 2024-12-08T04:01:46,347 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/info/ee08f55c0146486cbb84b1f1ced16afd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/info/ee08f55c0146486cbb84b1f1ced16afd 2024-12-08T04:01:46,349 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/info/ee08f55c0146486cbb84b1f1ced16afd, delay=994 ms 2024-12-08T04:01:46,350 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/info/ee08f55c0146486cbb84b1f1ced16afd, entries=15, sequenceid=15, filesize=7.6 K 2024-12-08T04:01:46,350 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/ns/fffd289f20954e5da06692674b90a0fd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/ns/fffd289f20954e5da06692674b90a0fd 2024-12-08T04:01:46,351 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/ns/fffd289f20954e5da06692674b90a0fd, delay=914 ms 2024-12-08T04:01:46,353 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/ns/fffd289f20954e5da06692674b90a0fd, entries=2, sequenceid=15, filesize=5.0 K 2024-12-08T04:01:46,353 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/table/9bf1843f6b7a4162ae69ae15b13f5cf1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/table/9bf1843f6b7a4162ae69ae15b13f5cf1 2024-12-08T04:01:46,354 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/table/9bf1843f6b7a4162ae69ae15b13f5cf1, delay=975 ms 2024-12-08T04:01:46,356 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/table/9bf1843f6b7a4162ae69ae15b13f5cf1, entries=2, sequenceid=15, filesize=5.3 K 2024-12-08T04:01:46,357 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 83ms, sequenceid=15, compaction requested=false 2024-12-08T04:01:46,357 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-08T04:01:46,358 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/info/ee08f55c0146486cbb84b1f1ced16afd 2024-12-08T04:01:46,359 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/ns/fffd289f20954e5da06692674b90a0fd 2024-12-08T04:01:46,359 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/table/9bf1843f6b7a4162ae69ae15b13f5cf1 2024-12-08T04:01:46,363 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-08T04:01:46,364 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T04:01:46,364 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T04:01:46,365 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733630506272Running coprocessor pre-close hooks at 1733630506272Disabling compacts and flushes for region at 1733630506272Disabling writes for close at 1733630506273 (+1 ms)Obtaining lock to block concurrent updates at 1733630506273Preparing flush snapshotting stores in 1588230740 at 1733630506273Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733630506274 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733630506274Flushing 1588230740/info: creating writer at 1733630506275 (+1 ms)Flushing 1588230740/info: appending metadata at 1733630506298 (+23 ms)Flushing 1588230740/info: closing flushed file at 1733630506298Flushing 1588230740/ns: creating writer at 1733630506305 (+7 ms)Flushing 1588230740/ns: appending metadata at 1733630506323 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1733630506323Flushing 1588230740/table: creating writer at 1733630506328 (+5 ms)Flushing 1588230740/table: appending metadata at 1733630506343 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733630506344 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e0b5d96: reopening flushed file at 1733630506347 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e75df60: reopening flushed file at 1733630506350 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@267bf997: reopening flushed file at 1733630506353 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 83ms, sequenceid=15, compaction requested=false at 1733630506357 (+4 ms)Writing region close event to WAL at 1733630506360 (+3 ms)Running coprocessor post-close hooks at 1733630506364 (+4 ms)Closed at 1733630506364 2024-12-08T04:01:46,365 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T04:01:46,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:46,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39845-0x10003a3dc860002, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:46,413 INFO [RS:1;32df4b4bb439:39845 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T04:01:46,413 INFO [RS:1;32df4b4bb439:39845 {}] regionserver.HRegionServer(1031): Exiting; stopping=32df4b4bb439,39845,1733630501515; zookeeper connection closed. 2024-12-08T04:01:46,413 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@207bbb3c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@207bbb3c 2024-12-08T04:01:46,473 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer(976): stopping server 32df4b4bb439,37281,1733630501379; all regions closed. 2024-12-08T04:01:46,478 DEBUG [RS:0;32df4b4bb439:37281 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/oldWALs 2024-12-08T04:01:46,478 INFO [RS:0;32df4b4bb439:37281 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32df4b4bb439%2C37281%2C1733630501379.meta:.meta(num 1733630503485) 2024-12-08T04:01:46,482 DEBUG [RS:0;32df4b4bb439:37281 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/oldWALs 2024-12-08T04:01:46,482 INFO [RS:0;32df4b4bb439:37281 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32df4b4bb439%2C37281%2C1733630501379:(num 1733630503141) 2024-12-08T04:01:46,482 DEBUG [RS:0;32df4b4bb439:37281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:46,482 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T04:01:46,483 INFO [RS:0;32df4b4bb439:37281 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T04:01:46,483 INFO [RS:0;32df4b4bb439:37281 {}] hbase.ChoreService(370): Chore service for: regionserver/32df4b4bb439:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-08T04:01:46,483 INFO [RS:0;32df4b4bb439:37281 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T04:01:46,483 INFO [regionserver/32df4b4bb439:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T04:01:46,484 INFO [RS:0;32df4b4bb439:37281 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37281 2024-12-08T04:01:46,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T04:01:46,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/32df4b4bb439,37281,1733630501379 2024-12-08T04:01:46,508 INFO [RS:0;32df4b4bb439:37281 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T04:01:46,509 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [32df4b4bb439,37281,1733630501379] 2024-12-08T04:01:46,524 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/32df4b4bb439,37281,1733630501379 already deleted, retry=false 2024-12-08T04:01:46,525 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 32df4b4bb439,37281,1733630501379 expired; onlineServers=0 2024-12-08T04:01:46,525 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '32df4b4bb439,35353,1733630500581' ***** 2024-12-08T04:01:46,525 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-08T04:01:46,525 INFO [M:0;32df4b4bb439:35353 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T04:01:46,525 INFO [M:0;32df4b4bb439:35353 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T04:01:46,525 DEBUG [M:0;32df4b4bb439:35353 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-08T04:01:46,525 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-08T04:01:46,525 DEBUG [M:0;32df4b4bb439:35353 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-08T04:01:46,526 DEBUG [master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.large.0-1733630502751 {}] cleaner.HFileCleaner(306): Exit Thread[master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.large.0-1733630502751,5,FailOnTimeoutGroup] 2024-12-08T04:01:46,526 DEBUG [master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.small.0-1733630502752 {}] cleaner.HFileCleaner(306): Exit Thread[master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.small.0-1733630502752,5,FailOnTimeoutGroup] 2024-12-08T04:01:46,526 INFO [M:0;32df4b4bb439:35353 {}] hbase.ChoreService(370): Chore service for: master/32df4b4bb439:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-08T04:01:46,526 INFO [M:0;32df4b4bb439:35353 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T04:01:46,526 DEBUG [M:0;32df4b4bb439:35353 {}] master.HMaster(1795): Stopping service threads 2024-12-08T04:01:46,526 INFO [M:0;32df4b4bb439:35353 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-08T04:01:46,526 INFO [M:0;32df4b4bb439:35353 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T04:01:46,527 INFO [M:0;32df4b4bb439:35353 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-08T04:01:46,527 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-08T04:01:46,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-08T04:01:46,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:46,537 DEBUG [M:0;32df4b4bb439:35353 {}] zookeeper.ZKUtil(347): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-08T04:01:46,537 WARN [M:0;32df4b4bb439:35353 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-08T04:01:46,538 INFO [M:0;32df4b4bb439:35353 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/.lastflushedseqids 2024-12-08T04:01:46,550 INFO [M:0;32df4b4bb439:35353 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-08T04:01:46,550 INFO [M:0;32df4b4bb439:35353 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-08T04:01:46,551 DEBUG [M:0;32df4b4bb439:35353 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T04:01:46,551 INFO [M:0;32df4b4bb439:35353 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:46,551 DEBUG [M:0;32df4b4bb439:35353 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:46,551 DEBUG [M:0;32df4b4bb439:35353 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T04:01:46,551 DEBUG [M:0;32df4b4bb439:35353 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:46,551 INFO [M:0;32df4b4bb439:35353 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-12-08T04:01:46,561 DEBUG [M:0;32df4b4bb439:35353 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e1e8b94eeb7d4ab4aa11033d09acf78a 2024-12-08T04:01:46,573 DEBUG [M:0;32df4b4bb439:35353 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e1e8b94eeb7d4ab4aa11033d09acf78a is 82, key is hbase:meta,,1/info:regioninfo/1733630503540/Put/seqid=0 2024-12-08T04:01:46,574 INFO [M:0;32df4b4bb439:35353 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e1e8b94eeb7d4ab4aa11033d09acf78a 2024-12-08T04:01:46,581 DEBUG [M:0;32df4b4bb439:35353 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e377e29396ed4986ad8859d3663fea2d 2024-12-08T04:01:46,593 DEBUG [M:0;32df4b4bb439:35353 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e377e29396ed4986ad8859d3663fea2d is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733630504266/Put/seqid=0 2024-12-08T04:01:46,594 INFO [M:0;32df4b4bb439:35353 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e377e29396ed4986ad8859d3663fea2d 2024-12-08T04:01:46,600 DEBUG [M:0;32df4b4bb439:35353 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e48b5eab582146b592a901f4d44b8792 2024-12-08T04:01:46,611 DEBUG [M:0;32df4b4bb439:35353 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e48b5eab582146b592a901f4d44b8792 is 69, key is 32df4b4bb439,37281,1733630501379/rs:state/1733630502864/Put/seqid=0 2024-12-08T04:01:46,611 INFO [M:0;32df4b4bb439:35353 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e48b5eab582146b592a901f4d44b8792 2024-12-08T04:01:46,613 DEBUG [M:0;32df4b4bb439:35353 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e1e8b94eeb7d4ab4aa11033d09acf78a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e1e8b94eeb7d4ab4aa11033d09acf78a 2024-12-08T04:01:46,614 INFO [M:0;32df4b4bb439:35353 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e1e8b94eeb7d4ab4aa11033d09acf78a, entries=8, sequenceid=84, filesize=5.5 K 2024-12-08T04:01:46,615 DEBUG [M:0;32df4b4bb439:35353 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e377e29396ed4986ad8859d3663fea2d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e377e29396ed4986ad8859d3663fea2d 2024-12-08T04:01:46,616 INFO [M:0;32df4b4bb439:35353 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e377e29396ed4986ad8859d3663fea2d, entries=9, sequenceid=84, filesize=6.5 K 2024-12-08T04:01:46,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:46,616 INFO [RS:0;32df4b4bb439:37281 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T04:01:46,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37281-0x10003a3dc860001, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:46,616 INFO [RS:0;32df4b4bb439:37281 {}] regionserver.HRegionServer(1031): Exiting; stopping=32df4b4bb439,37281,1733630501379; zookeeper connection closed. 2024-12-08T04:01:46,617 DEBUG [M:0;32df4b4bb439:35353 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e48b5eab582146b592a901f4d44b8792 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e48b5eab582146b592a901f4d44b8792 2024-12-08T04:01:46,617 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7a96b635 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7a96b635 2024-12-08T04:01:46,617 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-08T04:01:46,618 INFO [M:0;32df4b4bb439:35353 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e48b5eab582146b592a901f4d44b8792, entries=2, sequenceid=84, filesize=5.1 K 2024-12-08T04:01:46,619 INFO [M:0;32df4b4bb439:35353 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=84, compaction requested=false 2024-12-08T04:01:46,621 INFO [M:0;32df4b4bb439:35353 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:46,621 DEBUG [M:0;32df4b4bb439:35353 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733630506551Disabling compacts and flushes for region at 1733630506551Disabling writes for close at 1733630506551Obtaining lock to block concurrent updates at 1733630506551Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733630506551Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733630506552 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733630506553 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733630506553Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733630506573 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733630506573Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733630506576 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733630506592 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733630506592Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733630506595 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733630506610 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733630506610Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3885341d: reopening flushed file at 1733630506613 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cc42109: reopening flushed file at 1733630506615 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b4df5a1: reopening flushed file at 1733630506616 (+1 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=84, compaction requested=false at 1733630506619 (+3 ms)Writing region close event to WAL at 1733630506621 (+2 ms)Closed at 1733630506621 2024-12-08T04:01:46,623 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T04:01:46,623 INFO [M:0;32df4b4bb439:35353 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-08T04:01:46,623 INFO [M:0;32df4b4bb439:35353 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35353 2024-12-08T04:01:46,623 INFO [M:0;32df4b4bb439:35353 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T04:01:46,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:46,745 INFO [M:0;32df4b4bb439:35353 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T04:01:46,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35353-0x10003a3dc860000, quorum=127.0.0.1:60499, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:46,751 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-08T04:01:46,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-08T04:01:46,771 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:46,777 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733630506169 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1733630504628 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1733630506351 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/32df4b4bb439:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/32df4b4bb439:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/32df4b4bb439:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733630506349 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: regionserver/32df4b4bb439:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=382 (was 372) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=17511 (was 17886) 2024-12-08T04:01:46,782 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=382, ProcessCount=11, AvailableMemoryMB=17511 2024-12-08T04:01:46,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/hadoop-log-dir so I do NOT create it in target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0 2024-12-08T04:01:46,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/hadoop-tmp-dir so I do NOT create it in target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0 2024-12-08T04:01:46,783 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:46,785 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/cluster_5eda0489-75e7-506a-4934-1414f6c3b1a4, deleteOnExit=true 2024-12-08T04:01:46,787 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/cluster_5eda0489-75e7-506a-4934-1414f6c3b1a4/zookeeper_0, clientPort=50055, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/cluster_5eda0489-75e7-506a-4934-1414f6c3b1a4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/cluster_5eda0489-75e7-506a-4934-1414f6c3b1a4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-08T04:01:46,788 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50055 2024-12-08T04:01:46,798 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68 with version=8 2024-12-08T04:01:46,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/hbase-staging 2024-12-08T04:01:46,800 INFO [Time-limited test {}] client.ConnectionUtils(128): master/32df4b4bb439:0 server-side Connection retries=45 2024-12-08T04:01:46,801 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:46,801 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:46,801 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T04:01:46,801 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:46,801 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T04:01:46,802 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-08T04:01:46,802 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T04:01:46,803 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42935 2024-12-08T04:01:46,804 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42935 connecting to ZooKeeper ensemble=127.0.0.1:50055 2024-12-08T04:01:46,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:429350x0, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T04:01:46,919 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42935-0x10003a3f5bf0000 connected 2024-12-08T04:01:47,008 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:47,008 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:47,009 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T04:01:47,010 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68, hbase.cluster.distributed=false 2024-12-08T04:01:47,013 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T04:01:47,013 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42935 2024-12-08T04:01:47,014 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42935 2024-12-08T04:01:47,014 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42935 2024-12-08T04:01:47,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42935 2024-12-08T04:01:47,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42935 2024-12-08T04:01:47,033 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/32df4b4bb439:0 server-side Connection retries=45 2024-12-08T04:01:47,033 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:47,033 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:47,033 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T04:01:47,033 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:47,033 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T04:01:47,034 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T04:01:47,034 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T04:01:47,034 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46837 2024-12-08T04:01:47,036 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46837 connecting to ZooKeeper ensemble=127.0.0.1:50055 2024-12-08T04:01:47,036 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:47,036 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:47,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:468370x0, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T04:01:47,047 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:468370x0, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T04:01:47,047 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46837-0x10003a3f5bf0001 connected 2024-12-08T04:01:47,047 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T04:01:47,048 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/bucket.cache 2024-12-08T04:01:47,048 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-08T04:01:47,049 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-08T04:01:47,049 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-08T04:01:47,049 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-08T04:01:47,049 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-08T04:01:47,049 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-08T04:01:47,050 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T04:01:47,050 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-08T04:01:47,050 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/bucket.persistence is complete. 2024-12-08T04:01:47,051 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T04:01:47,052 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T04:01:47,053 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46837 2024-12-08T04:01:47,053 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46837 2024-12-08T04:01:47,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46837 2024-12-08T04:01:47,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46837 2024-12-08T04:01:47,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46837 2024-12-08T04:01:47,072 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/32df4b4bb439:0 server-side Connection retries=45 2024-12-08T04:01:47,072 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:47,072 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:47,072 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T04:01:47,072 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:47,072 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T04:01:47,072 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T04:01:47,072 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T04:01:47,073 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45549 2024-12-08T04:01:47,074 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45549 connecting to ZooKeeper ensemble=127.0.0.1:50055 2024-12-08T04:01:47,074 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:47,074 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:47,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:455490x0, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T04:01:47,083 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:455490x0, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T04:01:47,083 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45549-0x10003a3f5bf0002 connected 2024-12-08T04:01:47,083 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T04:01:47,084 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/bucket.cache 2024-12-08T04:01:47,084 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-08T04:01:47,085 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-08T04:01:47,085 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-08T04:01:47,085 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-08T04:01:47,085 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-08T04:01:47,085 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-08T04:01:47,085 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-08T04:01:47,086 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/bucket.persistence is complete. 2024-12-08T04:01:47,086 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T04:01:47,087 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T04:01:47,088 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T04:01:47,088 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45549 2024-12-08T04:01:47,089 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45549 2024-12-08T04:01:47,091 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45549 2024-12-08T04:01:47,091 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45549 2024-12-08T04:01:47,093 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45549 2024-12-08T04:01:47,109 DEBUG [M:0;32df4b4bb439:42935 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;32df4b4bb439:42935 2024-12-08T04:01:47,109 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/32df4b4bb439,42935,1733630506800 2024-12-08T04:01:47,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T04:01:47,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T04:01:47,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T04:01:47,116 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/32df4b4bb439,42935,1733630506800 2024-12-08T04:01:47,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T04:01:47,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T04:01:47,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,125 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T04:01:47,125 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/32df4b4bb439,42935,1733630506800 from backup master directory 2024-12-08T04:01:47,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T04:01:47,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T04:01:47,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/32df4b4bb439,42935,1733630506800 2024-12-08T04:01:47,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T04:01:47,132 WARN [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T04:01:47,132 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=32df4b4bb439,42935,1733630506800 2024-12-08T04:01:47,134 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/hbase.id] with ID: dcc340f8-585b-49ce-8a77-88808c07633b 2024-12-08T04:01:47,134 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/.tmp/hbase.id 2024-12-08T04:01:47,137 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/hbase.id] 2024-12-08T04:01:47,168 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:47,168 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-08T04:01:47,170 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-08T04:01:47,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,187 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T04:01:47,188 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-08T04:01:47,188 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T04:01:47,204 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store 2024-12-08T04:01:47,210 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:47,210 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T04:01:47,210 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:47,210 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:47,210 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T04:01:47,211 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:47,211 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:47,211 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733630507210Disabling compacts and flushes for region at 1733630507210Disabling writes for close at 1733630507211 (+1 ms)Writing region close event to WAL at 1733630507211Closed at 1733630507211 2024-12-08T04:01:47,211 WARN [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/.initializing 2024-12-08T04:01:47,211 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/WALs/32df4b4bb439,42935,1733630506800 2024-12-08T04:01:47,217 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T04:01:47,221 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32df4b4bb439%2C42935%2C1733630506800, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/WALs/32df4b4bb439,42935,1733630506800, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/oldWALs, maxLogs=10 2024-12-08T04:01:47,237 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/WALs/32df4b4bb439,42935,1733630506800/32df4b4bb439%2C42935%2C1733630506800.1733630507221 2024-12-08T04:01:47,237 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T04:01:47,237 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-08T04:01:47,238 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:47,238 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:47,238 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:47,239 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:47,242 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-08T04:01:47,242 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:47,243 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:47,243 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:47,245 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-08T04:01:47,246 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:47,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:47,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:47,249 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-08T04:01:47,249 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:47,249 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:47,249 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:47,252 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-08T04:01:47,253 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:47,253 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:47,253 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:47,254 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:47,254 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:47,254 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:47,254 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:47,254 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T04:01:47,255 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T04:01:47,260 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T04:01:47,260 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68370568, jitterRate=0.018800854682922363}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T04:01:47,261 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733630507238Initializing all the Stores at 1733630507238Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630507238Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630507238Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630507238Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630507238Cleaning up temporary data from old regions at 1733630507254 (+16 ms)Region opened successfully at 1733630507261 (+7 ms) 2024-12-08T04:01:47,262 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-08T04:01:47,265 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e14775a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32df4b4bb439/172.17.0.2:0 2024-12-08T04:01:47,266 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-08T04:01:47,266 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-08T04:01:47,266 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-08T04:01:47,266 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-08T04:01:47,267 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-08T04:01:47,267 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-08T04:01:47,267 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-08T04:01:47,269 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-08T04:01:47,270 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-08T04:01:47,288 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-08T04:01:47,288 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-08T04:01:47,289 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-08T04:01:47,299 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-08T04:01:47,299 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-08T04:01:47,301 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-08T04:01:47,307 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-08T04:01:47,309 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-08T04:01:47,315 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-08T04:01:47,319 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-08T04:01:47,329 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-08T04:01:47,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T04:01:47,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T04:01:47,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T04:01:47,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,341 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=32df4b4bb439,42935,1733630506800, sessionid=0x10003a3f5bf0000, setting cluster-up flag (Was=false) 2024-12-08T04:01:47,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,382 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-08T04:01:47,384 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32df4b4bb439,42935,1733630506800 2024-12-08T04:01:47,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,432 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-08T04:01:47,434 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32df4b4bb439,42935,1733630506800 2024-12-08T04:01:47,434 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-08T04:01:47,436 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-08T04:01:47,436 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-08T04:01:47,436 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-08T04:01:47,436 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 32df4b4bb439,42935,1733630506800 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-08T04:01:47,438 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/32df4b4bb439:0, corePoolSize=5, maxPoolSize=5 2024-12-08T04:01:47,438 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/32df4b4bb439:0, corePoolSize=5, maxPoolSize=5 2024-12-08T04:01:47,438 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/32df4b4bb439:0, corePoolSize=5, maxPoolSize=5 2024-12-08T04:01:47,438 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/32df4b4bb439:0, corePoolSize=5, maxPoolSize=5 2024-12-08T04:01:47,438 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/32df4b4bb439:0, corePoolSize=10, maxPoolSize=10 2024-12-08T04:01:47,438 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,438 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/32df4b4bb439:0, corePoolSize=2, maxPoolSize=2 2024-12-08T04:01:47,438 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,439 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733630537439 2024-12-08T04:01:47,439 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-08T04:01:47,439 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-08T04:01:47,439 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-08T04:01:47,439 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-08T04:01:47,439 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-08T04:01:47,439 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-08T04:01:47,440 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,440 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T04:01:47,440 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-08T04:01:47,440 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-08T04:01:47,440 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-08T04:01:47,440 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-08T04:01:47,440 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-08T04:01:47,440 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-08T04:01:47,441 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:47,441 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.large.0-1733630507440,5,FailOnTimeoutGroup] 2024-12-08T04:01:47,441 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-08T04:01:47,442 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.small.0-1733630507441,5,FailOnTimeoutGroup] 2024-12-08T04:01:47,442 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,442 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-08T04:01:47,442 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,442 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,458 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-08T04:01:47,458 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68 2024-12-08T04:01:47,466 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:47,467 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-08T04:01:47,470 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T04:01:47,470 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:47,470 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:47,470 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-08T04:01:47,474 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T04:01:47,474 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:47,474 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:47,474 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-08T04:01:47,477 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T04:01:47,477 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:47,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:47,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-08T04:01:47,480 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T04:01:47,480 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:47,481 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:47,481 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T04:01:47,481 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740 2024-12-08T04:01:47,482 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740 2024-12-08T04:01:47,482 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T04:01:47,482 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T04:01:47,482 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T04:01:47,483 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T04:01:47,488 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T04:01:47,488 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68292071, jitterRate=0.017631158232688904}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T04:01:47,489 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733630507466Initializing all the Stores at 1733630507467 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630507467Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630507467Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630507467Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630507467Cleaning up temporary data from old regions at 1733630507482 (+15 ms)Region opened successfully at 1733630507489 (+7 ms) 2024-12-08T04:01:47,489 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T04:01:47,489 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T04:01:47,489 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T04:01:47,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T04:01:47,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T04:01:47,490 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T04:01:47,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733630507489Disabling compacts and flushes for region at 1733630507489Disabling writes for close at 1733630507490 (+1 ms)Writing region close event to WAL at 1733630507490Closed at 1733630507490 2024-12-08T04:01:47,491 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T04:01:47,491 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-08T04:01:47,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T04:01:47,492 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T04:01:47,493 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-08T04:01:47,495 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer(746): ClusterId : dcc340f8-585b-49ce-8a77-88808c07633b 2024-12-08T04:01:47,495 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer(746): ClusterId : dcc340f8-585b-49ce-8a77-88808c07633b 2024-12-08T04:01:47,495 DEBUG [RS:0;32df4b4bb439:46837 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T04:01:47,495 DEBUG [RS:1;32df4b4bb439:45549 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T04:01:47,526 DEBUG [RS:0;32df4b4bb439:46837 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T04:01:47,526 DEBUG [RS:0;32df4b4bb439:46837 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T04:01:47,526 DEBUG [RS:1;32df4b4bb439:45549 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T04:01:47,526 DEBUG [RS:1;32df4b4bb439:45549 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T04:01:47,547 DEBUG [RS:0;32df4b4bb439:46837 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T04:01:47,547 DEBUG [RS:1;32df4b4bb439:45549 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T04:01:47,547 DEBUG [RS:0;32df4b4bb439:46837 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1eb77a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32df4b4bb439/172.17.0.2:0 2024-12-08T04:01:47,547 DEBUG [RS:1;32df4b4bb439:45549 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c7f5c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32df4b4bb439/172.17.0.2:0 2024-12-08T04:01:47,558 DEBUG [RS:0;32df4b4bb439:46837 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;32df4b4bb439:46837 2024-12-08T04:01:47,558 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T04:01:47,558 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T04:01:47,558 DEBUG [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T04:01:47,559 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer(2659): reportForDuty to master=32df4b4bb439,42935,1733630506800 with port=46837, startcode=1733630507033 2024-12-08T04:01:47,559 DEBUG [RS:0;32df4b4bb439:46837 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T04:01:47,561 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53639, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T04:01:47,562 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42935 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 32df4b4bb439,46837,1733630507033 2024-12-08T04:01:47,562 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42935 {}] master.ServerManager(517): Registering regionserver=32df4b4bb439,46837,1733630507033 2024-12-08T04:01:47,563 DEBUG [RS:1;32df4b4bb439:45549 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;32df4b4bb439:45549 2024-12-08T04:01:47,564 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T04:01:47,564 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T04:01:47,564 DEBUG [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T04:01:47,564 DEBUG [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68 2024-12-08T04:01:47,564 DEBUG [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-08T04:01:47,564 DEBUG [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T04:01:47,565 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer(2659): reportForDuty to master=32df4b4bb439,42935,1733630506800 with port=45549, startcode=1733630507071 2024-12-08T04:01:47,565 DEBUG [RS:1;32df4b4bb439:45549 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T04:01:47,567 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55077, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T04:01:47,568 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42935 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 32df4b4bb439,45549,1733630507071 2024-12-08T04:01:47,568 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42935 {}] master.ServerManager(517): Registering regionserver=32df4b4bb439,45549,1733630507071 2024-12-08T04:01:47,569 DEBUG [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68 2024-12-08T04:01:47,569 DEBUG [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-08T04:01:47,569 DEBUG [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T04:01:47,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T04:01:47,618 DEBUG [RS:0;32df4b4bb439:46837 {}] zookeeper.ZKUtil(111): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/32df4b4bb439,46837,1733630507033 2024-12-08T04:01:47,618 WARN [RS:0;32df4b4bb439:46837 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T04:01:47,618 INFO [RS:0;32df4b4bb439:46837 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T04:01:47,618 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [32df4b4bb439,46837,1733630507033] 2024-12-08T04:01:47,618 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [32df4b4bb439,45549,1733630507071] 2024-12-08T04:01:47,619 DEBUG [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,46837,1733630507033 2024-12-08T04:01:47,619 DEBUG [RS:1;32df4b4bb439:45549 {}] zookeeper.ZKUtil(111): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/32df4b4bb439,45549,1733630507071 2024-12-08T04:01:47,619 WARN [RS:1;32df4b4bb439:45549 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T04:01:47,619 INFO [RS:1;32df4b4bb439:45549 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T04:01:47,619 DEBUG [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,45549,1733630507071 2024-12-08T04:01:47,627 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T04:01:47,627 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T04:01:47,633 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T04:01:47,638 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T04:01:47,638 INFO [RS:0;32df4b4bb439:46837 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T04:01:47,638 INFO [RS:0;32df4b4bb439:46837 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,638 INFO [RS:1;32df4b4bb439:45549 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T04:01:47,638 INFO [RS:1;32df4b4bb439:45549 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,638 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T04:01:47,638 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T04:01:47,639 INFO [RS:1;32df4b4bb439:45549 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T04:01:47,639 INFO [RS:0;32df4b4bb439:46837 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T04:01:47,639 INFO [RS:1;32df4b4bb439:45549 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,639 INFO [RS:0;32df4b4bb439:46837 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,639 DEBUG [RS:1;32df4b4bb439:45549 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,639 DEBUG [RS:0;32df4b4bb439:46837 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,639 DEBUG [RS:1;32df4b4bb439:45549 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,639 DEBUG [RS:0;32df4b4bb439:46837 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,639 DEBUG [RS:1;32df4b4bb439:45549 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,639 DEBUG [RS:0;32df4b4bb439:46837 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,639 DEBUG [RS:1;32df4b4bb439:45549 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,640 DEBUG [RS:1;32df4b4bb439:45549 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,640 DEBUG [RS:0;32df4b4bb439:46837 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,640 DEBUG [RS:1;32df4b4bb439:45549 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/32df4b4bb439:0, corePoolSize=2, maxPoolSize=2 2024-12-08T04:01:47,640 DEBUG [RS:0;32df4b4bb439:46837 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,640 DEBUG [RS:1;32df4b4bb439:45549 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,640 DEBUG [RS:1;32df4b4bb439:45549 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,640 DEBUG [RS:0;32df4b4bb439:46837 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/32df4b4bb439:0, corePoolSize=2, maxPoolSize=2 2024-12-08T04:01:47,640 DEBUG [RS:1;32df4b4bb439:45549 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,640 DEBUG [RS:0;32df4b4bb439:46837 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,640 DEBUG [RS:1;32df4b4bb439:45549 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,640 DEBUG [RS:1;32df4b4bb439:45549 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,640 DEBUG [RS:0;32df4b4bb439:46837 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,640 DEBUG [RS:1;32df4b4bb439:45549 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,640 DEBUG [RS:0;32df4b4bb439:46837 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,640 DEBUG [RS:1;32df4b4bb439:45549 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/32df4b4bb439:0, corePoolSize=3, maxPoolSize=3 2024-12-08T04:01:47,640 DEBUG [RS:1;32df4b4bb439:45549 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0, corePoolSize=3, maxPoolSize=3 2024-12-08T04:01:47,640 DEBUG [RS:0;32df4b4bb439:46837 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,640 DEBUG [RS:0;32df4b4bb439:46837 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,640 DEBUG [RS:0;32df4b4bb439:46837 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:47,641 DEBUG [RS:0;32df4b4bb439:46837 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/32df4b4bb439:0, corePoolSize=3, maxPoolSize=3 2024-12-08T04:01:47,641 DEBUG [RS:0;32df4b4bb439:46837 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0, corePoolSize=3, maxPoolSize=3 2024-12-08T04:01:47,641 INFO [RS:1;32df4b4bb439:45549 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,641 INFO [RS:1;32df4b4bb439:45549 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,641 INFO [RS:0;32df4b4bb439:46837 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,641 INFO [RS:0;32df4b4bb439:46837 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,641 INFO [RS:1;32df4b4bb439:45549 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,641 INFO [RS:0;32df4b4bb439:46837 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,641 INFO [RS:1;32df4b4bb439:45549 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,641 INFO [RS:0;32df4b4bb439:46837 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,642 INFO [RS:0;32df4b4bb439:46837 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,642 INFO [RS:1;32df4b4bb439:45549 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,642 INFO [RS:0;32df4b4bb439:46837 {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,46837,1733630507033-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T04:01:47,642 INFO [RS:1;32df4b4bb439:45549 {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,45549,1733630507071-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T04:01:47,643 WARN [32df4b4bb439:42935 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T04:01:47,656 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T04:01:47,656 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T04:01:47,656 INFO [RS:1;32df4b4bb439:45549 {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,45549,1733630507071-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,656 INFO [RS:0;32df4b4bb439:46837 {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,46837,1733630507033-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,656 INFO [RS:0;32df4b4bb439:46837 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,656 INFO [RS:1;32df4b4bb439:45549 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,656 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.Replication(171): 32df4b4bb439,46837,1733630507033 started 2024-12-08T04:01:47,656 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.Replication(171): 32df4b4bb439,45549,1733630507071 started 2024-12-08T04:01:47,670 INFO [RS:0;32df4b4bb439:46837 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,671 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer(1482): Serving as 32df4b4bb439,46837,1733630507033, RpcServer on 32df4b4bb439/172.17.0.2:46837, sessionid=0x10003a3f5bf0001 2024-12-08T04:01:47,671 DEBUG [RS:0;32df4b4bb439:46837 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T04:01:47,671 DEBUG [RS:0;32df4b4bb439:46837 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 32df4b4bb439,46837,1733630507033 2024-12-08T04:01:47,671 DEBUG [RS:0;32df4b4bb439:46837 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32df4b4bb439,46837,1733630507033' 2024-12-08T04:01:47,671 DEBUG [RS:0;32df4b4bb439:46837 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T04:01:47,672 DEBUG [RS:0;32df4b4bb439:46837 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T04:01:47,672 DEBUG [RS:0;32df4b4bb439:46837 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T04:01:47,672 DEBUG [RS:0;32df4b4bb439:46837 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T04:01:47,672 DEBUG [RS:0;32df4b4bb439:46837 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 32df4b4bb439,46837,1733630507033 2024-12-08T04:01:47,672 DEBUG [RS:0;32df4b4bb439:46837 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32df4b4bb439,46837,1733630507033' 2024-12-08T04:01:47,672 DEBUG [RS:0;32df4b4bb439:46837 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T04:01:47,673 DEBUG [RS:0;32df4b4bb439:46837 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T04:01:47,673 INFO [RS:1;32df4b4bb439:45549 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:47,673 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer(1482): Serving as 32df4b4bb439,45549,1733630507071, RpcServer on 32df4b4bb439/172.17.0.2:45549, sessionid=0x10003a3f5bf0002 2024-12-08T04:01:47,673 DEBUG [RS:1;32df4b4bb439:45549 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T04:01:47,673 DEBUG [RS:1;32df4b4bb439:45549 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 32df4b4bb439,45549,1733630507071 2024-12-08T04:01:47,673 DEBUG [RS:0;32df4b4bb439:46837 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T04:01:47,674 DEBUG [RS:1;32df4b4bb439:45549 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32df4b4bb439,45549,1733630507071' 2024-12-08T04:01:47,674 DEBUG [RS:1;32df4b4bb439:45549 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T04:01:47,674 INFO [RS:0;32df4b4bb439:46837 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T04:01:47,674 INFO [RS:0;32df4b4bb439:46837 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T04:01:47,674 DEBUG [RS:1;32df4b4bb439:45549 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T04:01:47,674 DEBUG [RS:1;32df4b4bb439:45549 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T04:01:47,674 DEBUG [RS:1;32df4b4bb439:45549 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T04:01:47,675 DEBUG [RS:1;32df4b4bb439:45549 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 32df4b4bb439,45549,1733630507071 2024-12-08T04:01:47,675 DEBUG [RS:1;32df4b4bb439:45549 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32df4b4bb439,45549,1733630507071' 2024-12-08T04:01:47,675 DEBUG [RS:1;32df4b4bb439:45549 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T04:01:47,675 DEBUG [RS:1;32df4b4bb439:45549 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T04:01:47,675 DEBUG [RS:1;32df4b4bb439:45549 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T04:01:47,675 INFO [RS:1;32df4b4bb439:45549 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T04:01:47,675 INFO [RS:1;32df4b4bb439:45549 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T04:01:47,776 INFO [RS:0;32df4b4bb439:46837 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T04:01:47,778 INFO [RS:1;32df4b4bb439:45549 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T04:01:47,779 INFO [RS:0;32df4b4bb439:46837 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=32df4b4bb439%2C46837%2C1733630507033, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,46837,1733630507033, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/oldWALs, maxLogs=55 2024-12-08T04:01:47,779 INFO [RS:1;32df4b4bb439:45549 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=32df4b4bb439%2C45549%2C1733630507071, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,45549,1733630507071, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/oldWALs, maxLogs=55 2024-12-08T04:01:47,797 INFO [RS:0;32df4b4bb439:46837 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,46837,1733630507033/32df4b4bb439%2C46837%2C1733630507033.1733630507783 2024-12-08T04:01:47,797 INFO [RS:1;32df4b4bb439:45549 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,45549,1733630507071/32df4b4bb439%2C45549%2C1733630507071.1733630507783 2024-12-08T04:01:47,797 DEBUG [RS:0;32df4b4bb439:46837 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T04:01:47,797 DEBUG [RS:1;32df4b4bb439:45549 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T04:01:47,893 DEBUG [32df4b4bb439:42935 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-08T04:01:47,894 DEBUG [32df4b4bb439:42935 {}] balancer.BalancerClusterState(204): Hosts are {32df4b4bb439=0} racks are {/default-rack=0} 2024-12-08T04:01:47,895 DEBUG [32df4b4bb439:42935 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T04:01:47,895 DEBUG [32df4b4bb439:42935 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T04:01:47,895 DEBUG [32df4b4bb439:42935 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T04:01:47,895 DEBUG [32df4b4bb439:42935 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T04:01:47,895 INFO [32df4b4bb439:42935 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T04:01:47,895 INFO [32df4b4bb439:42935 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T04:01:47,895 DEBUG [32df4b4bb439:42935 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T04:01:47,896 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=32df4b4bb439,45549,1733630507071 2024-12-08T04:01:47,896 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32df4b4bb439,45549,1733630507071, state=OPENING 2024-12-08T04:01:47,907 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-08T04:01:47,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:47,916 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T04:01:47,916 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:47,916 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:47,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:47,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=32df4b4bb439,45549,1733630507071}] 2024-12-08T04:01:48,071 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T04:01:48,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36557, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T04:01:48,082 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T04:01:48,083 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T04:01:48,083 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T04:01:48,084 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=32df4b4bb439%2C45549%2C1733630507071.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,45549,1733630507071, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/oldWALs, maxLogs=55 2024-12-08T04:01:48,101 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,45549,1733630507071/32df4b4bb439%2C45549%2C1733630507071.meta.1733630508084.meta 2024-12-08T04:01:48,101 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T04:01:48,102 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T04:01:48,102 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T04:01:48,102 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T04:01:48,102 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T04:01:48,102 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T04:01:48,102 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:48,103 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T04:01:48,103 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T04:01:48,103 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-08T04:01:48,104 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T04:01:48,104 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:48,104 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:48,104 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-08T04:01:48,105 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T04:01:48,105 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:48,105 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:48,105 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-08T04:01:48,106 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T04:01:48,106 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:48,106 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:48,106 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-08T04:01:48,107 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T04:01:48,107 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:48,107 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:48,107 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T04:01:48,108 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740 2024-12-08T04:01:48,108 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740 2024-12-08T04:01:48,108 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T04:01:48,108 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T04:01:48,108 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T04:01:48,109 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T04:01:48,109 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63958606, jitterRate=-0.04694250226020813}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T04:01:48,109 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T04:01:48,110 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733630508103Writing region info on filesystem at 1733630508103Initializing all the Stores at 1733630508103Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630508103Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630508103Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630508103Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630508103Cleaning up temporary data from old regions at 1733630508108 (+5 ms)Running coprocessor post-open hooks at 1733630508109 (+1 ms)Region opened successfully at 1733630508110 (+1 ms) 2024-12-08T04:01:48,111 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733630508071 2024-12-08T04:01:48,114 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T04:01:48,114 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T04:01:48,114 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,45549,1733630507071 2024-12-08T04:01:48,115 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32df4b4bb439,45549,1733630507071, state=OPEN 2024-12-08T04:01:48,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T04:01:48,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T04:01:48,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T04:01:48,141 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=32df4b4bb439,45549,1733630507071 2024-12-08T04:01:48,141 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:48,141 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:48,141 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:48,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-08T04:01:48,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=32df4b4bb439,45549,1733630507071 in 225 msec 2024-12-08T04:01:48,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-08T04:01:48,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 652 msec 2024-12-08T04:01:48,145 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T04:01:48,145 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-08T04:01:48,148 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:48,149 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,45549,1733630507071, seqNum=-1] 2024-12-08T04:01:48,149 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:48,150 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50543, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:48,156 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 719 msec 2024-12-08T04:01:48,156 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733630508156, completionTime=-1 2024-12-08T04:01:48,156 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-08T04:01:48,156 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-08T04:01:48,159 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-08T04:01:48,159 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733630568159 2024-12-08T04:01:48,159 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733630628159 2024-12-08T04:01:48,159 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-08T04:01:48,159 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,42935,1733630506800-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:48,159 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,42935,1733630506800-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:48,159 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,42935,1733630506800-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:48,160 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-32df4b4bb439:42935, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:48,160 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:48,160 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:48,162 DEBUG [master/32df4b4bb439:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-08T04:01:48,164 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.031sec 2024-12-08T04:01:48,165 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-08T04:01:48,165 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-08T04:01:48,165 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-08T04:01:48,165 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-08T04:01:48,165 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-08T04:01:48,165 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,42935,1733630506800-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T04:01:48,165 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,42935,1733630506800-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-08T04:01:48,166 DEBUG [master/32df4b4bb439:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-08T04:01:48,166 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-08T04:01:48,166 INFO [master/32df4b4bb439:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,42935,1733630506800-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:48,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fb9de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:48,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32df4b4bb439,42935,-1 for getting cluster id 2024-12-08T04:01:48,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T04:01:48,199 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dcc340f8-585b-49ce-8a77-88808c07633b' 2024-12-08T04:01:48,199 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T04:01:48,200 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dcc340f8-585b-49ce-8a77-88808c07633b" 2024-12-08T04:01:48,200 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ef94f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:48,200 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32df4b4bb439,42935,-1] 2024-12-08T04:01:48,201 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T04:01:48,201 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:48,204 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T04:01:48,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d796be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T04:01:48,206 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:48,208 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,45549,1733630507071, seqNum=-1] 2024-12-08T04:01:48,208 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:48,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:48,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=32df4b4bb439,42935,1733630506800 2024-12-08T04:01:48,214 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T04:01:48,215 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 32df4b4bb439,42935,1733630506800 2024-12-08T04:01:48,215 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3410e240 2024-12-08T04:01:48,216 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T04:01:48,218 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T04:01:48,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733630508213', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T04:01:48,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733630508213 2024-12-08T04:01:48,221 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733630508213 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T04:01:48,221 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:48,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733630508213" procId is: 4 2024-12-08T04:01:48,221 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733630508213 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T04:01:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T04:01:48,229 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733630508213-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 298cc6209906f23162b4dc1c414380a9, NAME => 'testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733630508213', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68 2024-12-08T04:01:48,235 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733630508213-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:48,235 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733630508213-pool-0 {}] regionserver.HRegion(1722): Closing 298cc6209906f23162b4dc1c414380a9, disabling compactions & flushes 2024-12-08T04:01:48,235 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733630508213-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:48,235 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733630508213-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:48,235 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733630508213-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. after waiting 0 ms 2024-12-08T04:01:48,235 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733630508213-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:48,235 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733630508213-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:48,235 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733630508213-pool-0 {}] regionserver.HRegion(1676): Region close journal for 298cc6209906f23162b4dc1c414380a9: Waiting for close lock at 1733630508235Disabling compacts and flushes for region at 1733630508235Disabling writes for close at 1733630508235Writing region close event to WAL at 1733630508235Closed at 1733630508235 2024-12-08T04:01:48,236 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733630508213 execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T04:01:48,236 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733630508236"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733630508236"}]},"ts":"1733630508236"} 2024-12-08T04:01:48,239 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T04:01:48,240 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733630508213 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T04:01:48,240 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733630508213","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630508240"}]},"ts":"1733630508240"} 2024-12-08T04:01:48,242 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733630508213, state=ENABLING in hbase:meta 2024-12-08T04:01:48,242 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {32df4b4bb439=0} racks are {/default-rack=0} 2024-12-08T04:01:48,243 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T04:01:48,243 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T04:01:48,243 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T04:01:48,243 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T04:01:48,243 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T04:01:48,243 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T04:01:48,243 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T04:01:48,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733630508213, region=298cc6209906f23162b4dc1c414380a9, ASSIGN}] 2024-12-08T04:01:48,244 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733630508213, region=298cc6209906f23162b4dc1c414380a9, ASSIGN 2024-12-08T04:01:48,245 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733630508213, region=298cc6209906f23162b4dc1c414380a9, ASSIGN; state=OFFLINE, location=32df4b4bb439,45549,1733630507071; forceNewPlan=false, retain=false 2024-12-08T04:01:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T04:01:48,396 INFO [32df4b4bb439:42935 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-08T04:01:48,396 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=298cc6209906f23162b4dc1c414380a9, regionState=OPENING, regionLocation=32df4b4bb439,45549,1733630507071 2024-12-08T04:01:48,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733630508213, region=298cc6209906f23162b4dc1c414380a9, ASSIGN because future has completed 2024-12-08T04:01:48,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 298cc6209906f23162b4dc1c414380a9, server=32df4b4bb439,45549,1733630507071}] 2024-12-08T04:01:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T04:01:48,558 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:48,559 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 298cc6209906f23162b4dc1c414380a9, NAME => 'testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9.', STARTKEY => '', ENDKEY => ''} 2024-12-08T04:01:48,559 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733630508213 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:48,559 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:48,560 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:48,560 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:48,561 INFO [StoreOpener-298cc6209906f23162b4dc1c414380a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:48,567 INFO [StoreOpener-298cc6209906f23162b4dc1c414380a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 298cc6209906f23162b4dc1c414380a9 columnFamilyName family 2024-12-08T04:01:48,568 DEBUG [StoreOpener-298cc6209906f23162b4dc1c414380a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:48,568 INFO [StoreOpener-298cc6209906f23162b4dc1c414380a9-1 {}] regionserver.HStore(327): Store=298cc6209906f23162b4dc1c414380a9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:48,568 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:48,569 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:48,569 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:48,569 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:48,569 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:48,570 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:48,576 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T04:01:48,576 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 298cc6209906f23162b4dc1c414380a9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67954737, jitterRate=0.012604489922523499}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T04:01:48,576 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:48,577 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 298cc6209906f23162b4dc1c414380a9: Running coprocessor pre-open hook at 1733630508560Writing region info on filesystem at 1733630508560Initializing all the Stores at 1733630508560Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630508560Cleaning up temporary data from old regions at 1733630508570 (+10 ms)Running coprocessor post-open hooks at 1733630508576 (+6 ms)Region opened successfully at 1733630508577 (+1 ms) 2024-12-08T04:01:48,578 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9., pid=6, masterSystemTime=1733630508554 2024-12-08T04:01:48,580 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:48,580 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:48,580 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=298cc6209906f23162b4dc1c414380a9, regionState=OPEN, openSeqNum=2, regionLocation=32df4b4bb439,45549,1733630507071 2024-12-08T04:01:48,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 298cc6209906f23162b4dc1c414380a9, server=32df4b4bb439,45549,1733630507071 because future has completed 2024-12-08T04:01:48,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-08T04:01:48,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 298cc6209906f23162b4dc1c414380a9, server=32df4b4bb439,45549,1733630507071 in 183 msec 2024-12-08T04:01:48,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-08T04:01:48,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733630508213, region=298cc6209906f23162b4dc1c414380a9, ASSIGN in 343 msec 2024-12-08T04:01:48,588 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733630508213 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T04:01:48,588 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733630508213","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733630508588"}]},"ts":"1733630508588"} 2024-12-08T04:01:48,590 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733630508213, state=ENABLED in hbase:meta 2024-12-08T04:01:48,591 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733630508213 execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T04:01:48,593 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733630508213 in 372 msec 2024-12-08T04:01:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T04:01:48,851 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733630508213 completed 2024-12-08T04:01:48,852 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733630508213 get assigned. Timeout = 60000ms 2024-12-08T04:01:48,852 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T04:01:48,855 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733630508213 assigned to meta. Checking AM states. 2024-12-08T04:01:48,855 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T04:01:48,856 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733630508213 assigned. 2024-12-08T04:01:48,860 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733630508213', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9., hostname=32df4b4bb439,45549,1733630507071, seqNum=2] 2024-12-08T04:01:48,864 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 298cc6209906f23162b4dc1c414380a9 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-08T04:01:48,872 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9b7c5317eb8442d3ac169926c6d671dc 2024-12-08T04:01:48,884 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/.tmp/family/9b7c5317eb8442d3ac169926c6d671dc is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-08T04:01:48,885 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/.tmp/family/9b7c5317eb8442d3ac169926c6d671dc 2024-12-08T04:01:48,888 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/.tmp/family/9b7c5317eb8442d3ac169926c6d671dc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/family/9b7c5317eb8442d3ac169926c6d671dc 2024-12-08T04:01:48,890 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/family/9b7c5317eb8442d3ac169926c6d671dc, delay=958 ms 2024-12-08T04:01:48,891 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/family/9b7c5317eb8442d3ac169926c6d671dc, entries=2, sequenceid=6, filesize=4.9 K 2024-12-08T04:01:48,892 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 298cc6209906f23162b4dc1c414380a9 in 28ms, sequenceid=6, compaction requested=false 2024-12-08T04:01:48,892 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733630508213' 2024-12-08T04:01:48,893 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 298cc6209906f23162b4dc1c414380a9: 2024-12-08T04:01:49,017 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T04:01:49,849 DEBUG [hfile-prefetch-1733630506169 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9b7c5317eb8442d3ac169926c6d671dc_0. 2024-12-08T04:01:49,850 DEBUG [hfile-prefetch-1733630506169 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9b7c5317eb8442d3ac169926c6d671dc_117. 2024-12-08T04:01:49,850 DEBUG [hfile-prefetch-1733630506169 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/family/9b7c5317eb8442d3ac169926c6d671dc, with total blocks 2, and data blocks 1 2024-12-08T04:01:49,850 DEBUG [hfile-prefetch-1733630506169 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-08T04:01:49,850 DEBUG [hfile-prefetch-1733630506169 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/family/9b7c5317eb8442d3ac169926c6d671dc, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-08T04:01:49,850 DEBUG [hfile-prefetch-1733630506169 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/family/9b7c5317eb8442d3ac169926c6d671dc. Will sleep 100ms and try the verification again. 2024-12-08T04:01:49,893 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;32df4b4bb439:45549,5,FailOnTimeoutGroup] 2024-12-08T04:01:49,894 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '32df4b4bb439,45549,1733630507071' ***** 2024-12-08T04:01:49,894 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-12-08T04:01:49,894 INFO [RS:1;32df4b4bb439:45549 {}] bucket.BucketCache(1738): Disabling cache 2024-12-08T04:01:49,920 INFO [RS:1;32df4b4bb439:45549 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/bucket.persistence 2024-12-08T04:01:49,920 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-08T04:01:49,920 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-08T04:01:49,920 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-08T04:01:49,920 INFO [RS:1;32df4b4bb439:45549 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-08T04:01:49,920 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-08T04:01:49,930 DEBUG [RS:1;32df4b4bb439:45549 {}] bucket.BucketCache(1487): Thread RS:1;32df4b4bb439:45549 started persisting bucket cache to file 2024-12-08T04:01:49,931 DEBUG [RS:1;32df4b4bb439:45549 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T04:01:49,931 DEBUG [RS:1;32df4b4bb439:45549 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-08T04:01:49,938 DEBUG [RS:1;32df4b4bb439:45549 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-08T04:01:49,938 DEBUG [RS:1;32df4b4bb439:45549 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733630509930 2024-12-08T04:01:49,938 DEBUG [RS:1;32df4b4bb439:45549 {}] bucket.BucketCache(1509): Thread RS:1;32df4b4bb439:45549 finished persisting bucket cache to file, renaming 2024-12-08T04:01:49,938 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T04:01:49,939 INFO [RS:1;32df4b4bb439:45549 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T04:01:49,939 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T04:01:49,939 INFO [RS:1;32df4b4bb439:45549 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T04:01:49,939 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer(3091): Received CLOSE for 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:49,940 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer(959): stopping server 32df4b4bb439,45549,1733630507071 2024-12-08T04:01:49,940 INFO [RS:1;32df4b4bb439:45549 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T04:01:49,940 INFO [RS:1;32df4b4bb439:45549 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;32df4b4bb439:45549. 2024-12-08T04:01:49,940 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-08T04:01:49,940 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 298cc6209906f23162b4dc1c414380a9, disabling compactions & flushes 2024-12-08T04:01:49,940 DEBUG [RS:1;32df4b4bb439:45549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T04:01:49,941 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:49,941 DEBUG [RS:1;32df4b4bb439:45549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:49,941 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:49,941 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T04:01:49,941 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. after waiting 0 ms 2024-12-08T04:01:49,941 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:49,941 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T04:01:49,941 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T04:01:49,941 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T04:01:49,942 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-08T04:01:49,942 DEBUG [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer(1325): Online Regions={298cc6209906f23162b4dc1c414380a9=testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9., 1588230740=hbase:meta,,1.1588230740} 2024-12-08T04:01:49,942 DEBUG [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:49,942 DEBUG [StoreFileCloser-298cc6209906f23162b4dc1c414380a9-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/family/9b7c5317eb8442d3ac169926c6d671dc 2024-12-08T04:01:49,942 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-08T04:01:49,942 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T04:01:49,942 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T04:01:49,942 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T04:01:49,943 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T04:01:49,943 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T04:01:49,943 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-12-08T04:01:49,944 INFO [regionserver/32df4b4bb439:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T04:01:49,945 WARN [hfile-prefetch-1733630506169 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/family/9b7c5317eb8442d3ac169926c6d671dc, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-08T04:01:49,946 DEBUG [hfile-prefetch-1733630506169 {}] hfile.PrefetchExecutor(121): Prefetch completed for 9b7c5317eb8442d3ac169926c6d671dc 2024-12-08T04:01:49,947 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-08T04:01:49,948 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:49,948 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 298cc6209906f23162b4dc1c414380a9: Waiting for close lock at 1733630509940Running coprocessor pre-close hooks at 1733630509940Disabling compacts and flushes for region at 1733630509940Disabling writes for close at 1733630509941 (+1 ms)Writing region close event to WAL at 1733630509942 (+1 ms)Running coprocessor post-close hooks at 1733630509948 (+6 ms)Closed at 1733630509948 2024-12-08T04:01:49,948 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:49,951 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-08T04:01:49,951 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-08T04:01:49,952 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T04:01:49,952 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-08T04:01:49,955 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-08T04:01:49,955 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 14241f36394c430a8993febc097057ef 2024-12-08T04:01:49,955 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733630509951 2024-12-08T04:01:49,955 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-08T04:01:49,968 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/info/14241f36394c430a8993febc097057ef is 209, key is testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9./info:regioninfo/1733630508580/Put/seqid=0 2024-12-08T04:01:49,969 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/info/14241f36394c430a8993febc097057ef 2024-12-08T04:01:49,976 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1c784d4fa2c345dc95e7056b8cc0023b 2024-12-08T04:01:49,986 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/ns/1c784d4fa2c345dc95e7056b8cc0023b is 43, key is default/ns:d/1733630508151/Put/seqid=0 2024-12-08T04:01:49,987 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/ns/1c784d4fa2c345dc95e7056b8cc0023b 2024-12-08T04:01:49,994 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c6877af379ba471bb8b891140f1dbd10 2024-12-08T04:01:50,005 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/table/c6877af379ba471bb8b891140f1dbd10 is 80, key is testBlockEvictionOnGracefulStop1733630508213/table:state/1733630508588/Put/seqid=0 2024-12-08T04:01:50,006 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/table/c6877af379ba471bb8b891140f1dbd10 2024-12-08T04:01:50,008 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/info/14241f36394c430a8993febc097057ef as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/info/14241f36394c430a8993febc097057ef 2024-12-08T04:01:50,010 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/info/14241f36394c430a8993febc097057ef, delay=932 ms 2024-12-08T04:01:50,011 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/info/14241f36394c430a8993febc097057ef, entries=10, sequenceid=11, filesize=7.0 K 2024-12-08T04:01:50,011 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/ns/1c784d4fa2c345dc95e7056b8cc0023b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/ns/1c784d4fa2c345dc95e7056b8cc0023b 2024-12-08T04:01:50,013 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/ns/1c784d4fa2c345dc95e7056b8cc0023b, delay=996 ms 2024-12-08T04:01:50,014 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/ns/1c784d4fa2c345dc95e7056b8cc0023b, entries=2, sequenceid=11, filesize=5.0 K 2024-12-08T04:01:50,015 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/table/c6877af379ba471bb8b891140f1dbd10 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/table/c6877af379ba471bb8b891140f1dbd10 2024-12-08T04:01:50,016 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/table/c6877af379ba471bb8b891140f1dbd10, delay=946 ms 2024-12-08T04:01:50,017 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/table/c6877af379ba471bb8b891140f1dbd10, entries=2, sequenceid=11, filesize=5.3 K 2024-12-08T04:01:50,018 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=11, compaction requested=false 2024-12-08T04:01:50,019 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/info/14241f36394c430a8993febc097057ef 2024-12-08T04:01:50,019 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/ns/1c784d4fa2c345dc95e7056b8cc0023b 2024-12-08T04:01:50,020 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/table/c6877af379ba471bb8b891140f1dbd10 2024-12-08T04:01:50,024 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-08T04:01:50,024 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T04:01:50,024 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T04:01:50,024 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733630509942Running coprocessor pre-close hooks at 1733630509942Disabling compacts and flushes for region at 1733630509942Disabling writes for close at 1733630509943 (+1 ms)Obtaining lock to block concurrent updates at 1733630509943Preparing flush snapshotting stores in 1588230740 at 1733630509943Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1733630509944 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733630509944Flushing 1588230740/info: creating writer at 1733630509944Flushing 1588230740/info: appending metadata at 1733630509968 (+24 ms)Flushing 1588230740/info: closing flushed file at 1733630509968Flushing 1588230740/ns: creating writer at 1733630509971 (+3 ms)Flushing 1588230740/ns: appending metadata at 1733630509986 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733630509986Flushing 1588230740/table: creating writer at 1733630509989 (+3 ms)Flushing 1588230740/table: appending metadata at 1733630510005 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733630510005Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fe03459: reopening flushed file at 1733630510008 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20dfe011: reopening flushed file at 1733630510011 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fe58523: reopening flushed file at 1733630510014 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=11, compaction requested=false at 1733630510018 (+4 ms)Writing region close event to WAL at 1733630510020 (+2 ms)Running coprocessor post-close hooks at 1733630510024 (+4 ms)Closed at 1733630510024 2024-12-08T04:01:50,024 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T04:01:50,142 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer(976): stopping server 32df4b4bb439,45549,1733630507071; all regions closed. 2024-12-08T04:01:50,147 DEBUG [RS:1;32df4b4bb439:45549 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/oldWALs 2024-12-08T04:01:50,147 INFO [RS:1;32df4b4bb439:45549 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32df4b4bb439%2C45549%2C1733630507071.meta:.meta(num 1733630508084) 2024-12-08T04:01:50,150 DEBUG [RS:1;32df4b4bb439:45549 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/oldWALs 2024-12-08T04:01:50,150 INFO [RS:1;32df4b4bb439:45549 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32df4b4bb439%2C45549%2C1733630507071:(num 1733630507783) 2024-12-08T04:01:50,150 DEBUG [RS:1;32df4b4bb439:45549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:50,150 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T04:01:50,150 INFO [RS:1;32df4b4bb439:45549 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T04:01:50,150 INFO [RS:1;32df4b4bb439:45549 {}] hbase.ChoreService(370): Chore service for: regionserver/32df4b4bb439:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-08T04:01:50,150 INFO [RS:1;32df4b4bb439:45549 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T04:01:50,151 INFO [regionserver/32df4b4bb439:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T04:01:50,151 INFO [RS:1;32df4b4bb439:45549 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45549 2024-12-08T04:01:50,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T04:01:50,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/32df4b4bb439,45549,1733630507071 2024-12-08T04:01:50,188 INFO [RS:1;32df4b4bb439:45549 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T04:01:50,189 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [32df4b4bb439,45549,1733630507071] 2024-12-08T04:01:50,207 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/32df4b4bb439,45549,1733630507071 already deleted, retry=false 2024-12-08T04:01:50,208 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 32df4b4bb439,45549,1733630507071 on 32df4b4bb439,42935,1733630506800 2024-12-08T04:01:50,216 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 32df4b4bb439,45549,1733630507071, splitWal=true, meta=true 2024-12-08T04:01:50,218 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 32df4b4bb439,45549,1733630507071 (carryingMeta=true) 32df4b4bb439,45549,1733630507071/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@39d9465a[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-08T04:01:50,218 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 32df4b4bb439,45549,1733630507071, splitWal=true, meta=true 2024-12-08T04:01:50,219 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 32df4b4bb439,45549,1733630507071, splitWal=true, meta=true, isMeta: true 2024-12-08T04:01:50,219 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,45549,1733630507071-splitting 2024-12-08T04:01:50,220 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,45549,1733630507071-splitting dir is empty, no logs to split. 2024-12-08T04:01:50,220 INFO [PEWorker-1 {}] master.SplitWALManager(105): 32df4b4bb439,45549,1733630507071 WAL count=0, meta=true 2024-12-08T04:01:50,222 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,45549,1733630507071-splitting dir is empty, no logs to split. 2024-12-08T04:01:50,222 INFO [PEWorker-1 {}] master.SplitWALManager(105): 32df4b4bb439,45549,1733630507071 WAL count=0, meta=true 2024-12-08T04:01:50,222 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 32df4b4bb439,45549,1733630507071 WAL splitting is done? wals=0, meta=true 2024-12-08T04:01:50,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T04:01:50,224 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T04:01:50,225 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-08T04:01:50,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:50,299 INFO [RS:1;32df4b4bb439:45549 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T04:01:50,299 INFO [RS:1;32df4b4bb439:45549 {}] regionserver.HRegionServer(1031): Exiting; stopping=32df4b4bb439,45549,1733630507071; zookeeper connection closed. 2024-12-08T04:01:50,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45549-0x10003a3f5bf0002, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:50,300 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7f17cdee {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7f17cdee 2024-12-08T04:01:50,375 DEBUG [32df4b4bb439:42935 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-08T04:01:50,376 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=32df4b4bb439,46837,1733630507033 2024-12-08T04:01:50,377 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32df4b4bb439,46837,1733630507033, state=OPENING 2024-12-08T04:01:50,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T04:01:50,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T04:01:50,390 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T04:01:50,390 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:50,390 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:50,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=32df4b4bb439,46837,1733630507033}] 2024-12-08T04:01:50,409 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/32df4b4bb439:0 server-side Connection retries=45 2024-12-08T04:01:50,409 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:50,409 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:50,409 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T04:01:50,409 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T04:01:50,409 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T04:01:50,409 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T04:01:50,409 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T04:01:50,410 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39919 2024-12-08T04:01:50,411 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39919 connecting to ZooKeeper ensemble=127.0.0.1:50055 2024-12-08T04:01:50,411 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:50,411 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:50,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:399190x0, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T04:01:50,424 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-08T04:01:50,424 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:399190x0, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-08T04:01:50,424 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39919-0x10003a3f5bf0003 connected 2024-12-08T04:01:50,425 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T04:01:50,426 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/bucket.cache 2024-12-08T04:01:50,426 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-08T04:01:50,426 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-08T04:01:50,426 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-08T04:01:50,426 INFO [Thread-208 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-08T04:01:50,426 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-08T04:01:50,426 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T04:01:50,427 INFO [Thread-208 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-08T04:01:50,427 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39919-0x10003a3f5bf0003, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T04:01:50,427 INFO [Thread-208 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-08T04:01:50,429 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39919-0x10003a3f5bf0003, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T04:01:50,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39919 2024-12-08T04:01:50,430 DEBUG [Thread-208 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-08T04:01:50,430 INFO [Thread-208 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-08T04:01:50,430 INFO [Thread-208 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-08T04:01:50,430 INFO [Thread-208 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/bucket.persistence is complete. 2024-12-08T04:01:50,435 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39919 2024-12-08T04:01:50,436 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39919 2024-12-08T04:01:50,437 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39919 2024-12-08T04:01:50,437 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39919 2024-12-08T04:01:50,440 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.HRegionServer(746): ClusterId : dcc340f8-585b-49ce-8a77-88808c07633b 2024-12-08T04:01:50,440 DEBUG [RS:2;32df4b4bb439:39919 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T04:01:50,449 DEBUG [RS:2;32df4b4bb439:39919 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T04:01:50,450 DEBUG [RS:2;32df4b4bb439:39919 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T04:01:50,458 DEBUG [RS:2;32df4b4bb439:39919 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T04:01:50,458 DEBUG [RS:2;32df4b4bb439:39919 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4f4a76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32df4b4bb439/172.17.0.2:0 2024-12-08T04:01:50,468 DEBUG [RS:2;32df4b4bb439:39919 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;32df4b4bb439:39919 2024-12-08T04:01:50,468 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T04:01:50,468 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T04:01:50,468 DEBUG [RS:2;32df4b4bb439:39919 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T04:01:50,469 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.HRegionServer(2659): reportForDuty to master=32df4b4bb439,42935,1733630506800 with port=39919, startcode=1733630510408 2024-12-08T04:01:50,469 DEBUG [RS:2;32df4b4bb439:39919 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T04:01:50,471 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39327, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T04:01:50,472 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42935 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 32df4b4bb439,39919,1733630510408 2024-12-08T04:01:50,472 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42935 {}] master.ServerManager(517): Registering regionserver=32df4b4bb439,39919,1733630510408 2024-12-08T04:01:50,474 DEBUG [RS:2;32df4b4bb439:39919 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68 2024-12-08T04:01:50,474 DEBUG [RS:2;32df4b4bb439:39919 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-08T04:01:50,474 DEBUG [RS:2;32df4b4bb439:39919 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T04:01:50,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T04:01:50,482 DEBUG [RS:2;32df4b4bb439:39919 {}] zookeeper.ZKUtil(111): regionserver:39919-0x10003a3f5bf0003, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/32df4b4bb439,39919,1733630510408 2024-12-08T04:01:50,483 WARN [RS:2;32df4b4bb439:39919 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T04:01:50,483 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [32df4b4bb439,39919,1733630510408] 2024-12-08T04:01:50,483 INFO [RS:2;32df4b4bb439:39919 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T04:01:50,483 DEBUG [RS:2;32df4b4bb439:39919 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,39919,1733630510408 2024-12-08T04:01:50,487 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T04:01:50,489 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T04:01:50,489 INFO [RS:2;32df4b4bb439:39919 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T04:01:50,489 INFO [RS:2;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:50,489 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T04:01:50,490 INFO [RS:2;32df4b4bb439:39919 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T04:01:50,490 INFO [RS:2;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:50,490 DEBUG [RS:2;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:50,490 DEBUG [RS:2;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:50,490 DEBUG [RS:2;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:50,490 DEBUG [RS:2;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:50,490 DEBUG [RS:2;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:50,490 DEBUG [RS:2;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/32df4b4bb439:0, corePoolSize=2, maxPoolSize=2 2024-12-08T04:01:50,490 DEBUG [RS:2;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:50,490 DEBUG [RS:2;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:50,490 DEBUG [RS:2;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:50,490 DEBUG [RS:2;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:50,490 DEBUG [RS:2;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:50,490 DEBUG [RS:2;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/32df4b4bb439:0, corePoolSize=1, maxPoolSize=1 2024-12-08T04:01:50,490 DEBUG [RS:2;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/32df4b4bb439:0, corePoolSize=3, maxPoolSize=3 2024-12-08T04:01:50,491 DEBUG [RS:2;32df4b4bb439:39919 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/32df4b4bb439:0, corePoolSize=3, maxPoolSize=3 2024-12-08T04:01:50,492 INFO [RS:2;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:50,492 INFO [RS:2;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:50,492 INFO [RS:2;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:50,492 INFO [RS:2;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:50,492 INFO [RS:2;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:50,492 INFO [RS:2;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,39919,1733630510408-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T04:01:50,505 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T04:01:50,505 INFO [RS:2;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=32df4b4bb439,39919,1733630510408-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:50,505 INFO [RS:2;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:50,505 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.Replication(171): 32df4b4bb439,39919,1733630510408 started 2024-12-08T04:01:50,517 INFO [RS:2;32df4b4bb439:39919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T04:01:50,517 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.HRegionServer(1482): Serving as 32df4b4bb439,39919,1733630510408, RpcServer on 32df4b4bb439/172.17.0.2:39919, sessionid=0x10003a3f5bf0003 2024-12-08T04:01:50,517 DEBUG [RS:2;32df4b4bb439:39919 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T04:01:50,517 DEBUG [RS:2;32df4b4bb439:39919 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 32df4b4bb439,39919,1733630510408 2024-12-08T04:01:50,518 DEBUG [RS:2;32df4b4bb439:39919 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32df4b4bb439,39919,1733630510408' 2024-12-08T04:01:50,518 DEBUG [RS:2;32df4b4bb439:39919 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T04:01:50,518 DEBUG [RS:2;32df4b4bb439:39919 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T04:01:50,519 DEBUG [RS:2;32df4b4bb439:39919 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T04:01:50,519 DEBUG [RS:2;32df4b4bb439:39919 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T04:01:50,519 DEBUG [RS:2;32df4b4bb439:39919 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 32df4b4bb439,39919,1733630510408 2024-12-08T04:01:50,519 DEBUG [RS:2;32df4b4bb439:39919 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32df4b4bb439,39919,1733630510408' 2024-12-08T04:01:50,519 DEBUG [RS:2;32df4b4bb439:39919 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T04:01:50,519 DEBUG [RS:2;32df4b4bb439:39919 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T04:01:50,520 DEBUG [RS:2;32df4b4bb439:39919 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T04:01:50,520 INFO [RS:2;32df4b4bb439:39919 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T04:01:50,520 INFO [RS:2;32df4b4bb439:39919 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T04:01:50,543 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T04:01:50,545 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40979, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T04:01:50,547 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T04:01:50,547 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T04:01:50,548 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T04:01:50,549 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=32df4b4bb439%2C46837%2C1733630507033.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,46837,1733630507033, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/oldWALs, maxLogs=55 2024-12-08T04:01:50,565 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,46837,1733630507033/32df4b4bb439%2C46837%2C1733630507033.meta.1733630510549.meta 2024-12-08T04:01:50,565 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T04:01:50,566 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T04:01:50,566 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T04:01:50,566 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T04:01:50,566 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T04:01:50,566 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T04:01:50,566 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:50,566 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T04:01:50,567 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T04:01:50,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-08T04:01:50,568 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T04:01:50,568 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:50,570 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/info/14241f36394c430a8993febc097057ef, delay=950 ms 2024-12-08T04:01:50,571 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/info/14241f36394c430a8993febc097057ef 2024-12-08T04:01:50,571 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:50,571 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-08T04:01:50,572 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T04:01:50,572 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:50,573 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/ns/1c784d4fa2c345dc95e7056b8cc0023b, delay=904 ms 2024-12-08T04:01:50,574 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/ns/1c784d4fa2c345dc95e7056b8cc0023b 2024-12-08T04:01:50,575 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:50,575 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-08T04:01:50,575 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T04:01:50,575 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:50,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:50,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-08T04:01:50,576 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T04:01:50,576 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:50,578 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/table/c6877af379ba471bb8b891140f1dbd10, delay=988 ms 2024-12-08T04:01:50,579 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/table/c6877af379ba471bb8b891140f1dbd10 2024-12-08T04:01:50,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T04:01:50,579 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T04:01:50,580 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740 2024-12-08T04:01:50,580 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740 2024-12-08T04:01:50,580 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T04:01:50,580 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T04:01:50,581 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T04:01:50,581 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T04:01:50,582 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59399289, jitterRate=-0.11488161981105804}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T04:01:50,582 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T04:01:50,582 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733630510567Writing region info on filesystem at 1733630510567Initializing all the Stores at 1733630510567Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630510567Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630510567Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630510567Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733630510567Cleaning up temporary data from old regions at 1733630510580 (+13 ms)Running coprocessor post-open hooks at 1733630510582 (+2 ms)Region opened successfully at 1733630510582 2024-12-08T04:01:50,583 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1733630510542 2024-12-08T04:01:50,585 DEBUG [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T04:01:50,586 INFO [RS_OPEN_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T04:01:50,586 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=32df4b4bb439,46837,1733630507033 2024-12-08T04:01:50,586 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32df4b4bb439,46837,1733630507033, state=OPEN 2024-12-08T04:01:50,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T04:01:50,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a3f5bf0003, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T04:01:50,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T04:01:50,621 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=32df4b4bb439,46837,1733630507033 2024-12-08T04:01:50,621 INFO [RS:2;32df4b4bb439:39919 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T04:01:50,621 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:50,621 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:50,621 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T04:01:50,621 INFO [RS:2;32df4b4bb439:39919 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=32df4b4bb439%2C39919%2C1733630510408, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,39919,1733630510408, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/oldWALs, maxLogs=55 2024-12-08T04:01:50,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-08T04:01:50,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=32df4b4bb439,46837,1733630507033 in 231 msec 2024-12-08T04:01:50,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-08T04:01:50,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 401 msec 2024-12-08T04:01:50,627 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 32df4b4bb439,45549,1733630507071 had 2 regions 2024-12-08T04:01:50,628 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 32df4b4bb439,45549,1733630507071, splitWal=true, meta=true, isMeta: false 2024-12-08T04:01:50,629 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,45549,1733630507071-splitting dir is empty, no logs to split. 2024-12-08T04:01:50,629 INFO [PEWorker-3 {}] master.SplitWALManager(105): 32df4b4bb439,45549,1733630507071 WAL count=0, meta=false 2024-12-08T04:01:50,630 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,45549,1733630507071-splitting dir is empty, no logs to split. 2024-12-08T04:01:50,630 INFO [PEWorker-3 {}] master.SplitWALManager(105): 32df4b4bb439,45549,1733630507071 WAL count=0, meta=false 2024-12-08T04:01:50,630 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 32df4b4bb439,45549,1733630507071 WAL splitting is done? wals=0, meta=false 2024-12-08T04:01:50,632 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733630508213, region=298cc6209906f23162b4dc1c414380a9, ASSIGN}] 2024-12-08T04:01:50,633 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733630508213, region=298cc6209906f23162b4dc1c414380a9, ASSIGN 2024-12-08T04:01:50,634 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733630508213, region=298cc6209906f23162b4dc1c414380a9, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-08T04:01:50,639 INFO [RS:2;32df4b4bb439:39919 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/WALs/32df4b4bb439,39919,1733630510408/32df4b4bb439%2C39919%2C1733630510408.1733630510622 2024-12-08T04:01:50,640 DEBUG [RS:2;32df4b4bb439:39919 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-08T04:01:50,785 DEBUG [32df4b4bb439:42935 {}] balancer.BalancerClusterState(204): Hosts are {32df4b4bb439=0} racks are {/default-rack=0} 2024-12-08T04:01:50,785 DEBUG [32df4b4bb439:42935 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T04:01:50,785 DEBUG [32df4b4bb439:42935 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T04:01:50,785 DEBUG [32df4b4bb439:42935 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T04:01:50,785 DEBUG [32df4b4bb439:42935 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T04:01:50,785 INFO [32df4b4bb439:42935 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T04:01:50,785 INFO [32df4b4bb439:42935 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T04:01:50,785 DEBUG [32df4b4bb439:42935 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T04:01:50,786 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=298cc6209906f23162b4dc1c414380a9, regionState=OPENING, regionLocation=32df4b4bb439,46837,1733630507033 2024-12-08T04:01:50,787 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 32df4b4bb439:45549 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 32df4b4bb439/172.17.0.2:45549 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:50,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,45549,1733630507071, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,45549,1733630507071, seqNum=-1, error=java.net.ConnectException: Call to address=32df4b4bb439:45549 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 32df4b4bb439/172.17.0.2:45549 2024-12-08T04:01:50,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,45549,1733630507071, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-08T04:01:50,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,45549,1733630507071, seqNum=-1 from cache 2024-12-08T04:01:50,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 32df4b4bb439:45549 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 32df4b4bb439/172.17.0.2:45549 2024-12-08T04:01:50,900 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:50,900 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,46837,1733630507033, seqNum=-1] 2024-12-08T04:01:50,900 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T04:01:50,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34225, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T04:01:50,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733630508213, region=298cc6209906f23162b4dc1c414380a9, ASSIGN because future has completed 2024-12-08T04:01:50,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 298cc6209906f23162b4dc1c414380a9, server=32df4b4bb439,46837,1733630507033}] 2024-12-08T04:01:50,996 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-08T04:01:50,996 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-08T04:01:50,998 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T04:01:50,998 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-08T04:01:50,999 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-08T04:01:50,999 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-08T04:01:51,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-08T04:01:51,018 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T04:01:51,018 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T04:01:51,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:51,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:51,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T04:01:51,019 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-08T04:01:51,019 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1954655726, stopped=false 2024-12-08T04:01:51,019 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=32df4b4bb439,42935,1733630506800 2024-12-08T04:01:51,060 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:51,060 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 298cc6209906f23162b4dc1c414380a9, NAME => 'testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9.', STARTKEY => '', ENDKEY => ''} 2024-12-08T04:01:51,061 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733630508213 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:51,061 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T04:01:51,061 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:51,061 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:51,062 INFO [StoreOpener-298cc6209906f23162b4dc1c414380a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:51,063 INFO [StoreOpener-298cc6209906f23162b4dc1c414380a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 298cc6209906f23162b4dc1c414380a9 columnFamilyName family 2024-12-08T04:01:51,063 DEBUG [StoreOpener-298cc6209906f23162b4dc1c414380a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T04:01:51,064 DEBUG [StoreFileOpener-298cc6209906f23162b4dc1c414380a9-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/family/9b7c5317eb8442d3ac169926c6d671dc, delay=973 ms 2024-12-08T04:01:51,065 DEBUG [StoreOpener-298cc6209906f23162b4dc1c414380a9-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/family/9b7c5317eb8442d3ac169926c6d671dc 2024-12-08T04:01:51,066 INFO [StoreOpener-298cc6209906f23162b4dc1c414380a9-1 {}] regionserver.HStore(327): Store=298cc6209906f23162b4dc1c414380a9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T04:01:51,066 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:51,066 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:51,067 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:51,067 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:51,067 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:51,068 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:51,068 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 298cc6209906f23162b4dc1c414380a9; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73582797, jitterRate=0.09646911919116974}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T04:01:51,068 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:51,069 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 298cc6209906f23162b4dc1c414380a9: Running coprocessor pre-open hook at 1733630511061Writing region info on filesystem at 1733630511061Initializing all the Stores at 1733630511062 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733630511062Cleaning up temporary data from old regions at 1733630511067 (+5 ms)Running coprocessor post-open hooks at 1733630511068 (+1 ms)Region opened successfully at 1733630511069 (+1 ms) 2024-12-08T04:01:51,069 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9., pid=11, masterSystemTime=1733630511057 2024-12-08T04:01:51,072 DEBUG [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:51,072 INFO [RS_OPEN_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:51,072 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=298cc6209906f23162b4dc1c414380a9, regionState=OPEN, openSeqNum=10, regionLocation=32df4b4bb439,46837,1733630507033 2024-12-08T04:01:51,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T04:01:51,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a3f5bf0003, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T04:01:51,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T04:01:51,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a3f5bf0003, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:51,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:51,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:51,074 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T04:01:51,074 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T04:01:51,074 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T04:01:51,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:51,074 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T04:01:51,074 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '32df4b4bb439,46837,1733630507033' ***** 2024-12-08T04:01:51,074 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T04:01:51,074 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39919-0x10003a3f5bf0003, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T04:01:51,074 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '32df4b4bb439,39919,1733630510408' ***** 2024-12-08T04:01:51,074 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T04:01:51,075 INFO [RS:0;32df4b4bb439:46837 {}] bucket.BucketCache(1738): Disabling cache 2024-12-08T04:01:51,075 INFO [RS:2;32df4b4bb439:39919 {}] bucket.BucketCache(1738): Disabling cache 2024-12-08T04:01:51,075 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-08T04:01:51,075 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-08T04:01:51,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,46837,1733630507033, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,46837,1733630507033, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=32df4b4bb439:46837 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-08T04:01:51,075 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-08T04:01:51,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,46837,1733630507033, seqNum=-1 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-08T04:01:51,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,46837,1733630507033, seqNum=-1 from cache 2024-12-08T04:01:51,075 INFO [RS:0;32df4b4bb439:46837 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/bucket.persistence 2024-12-08T04:01:51,075 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-08T04:01:51,075 INFO [RS:2;32df4b4bb439:39919 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6185658b-76a8-ad73-c4f7-585b25a13ee0/bucket.persistence 2024-12-08T04:01:51,075 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-08T04:01:51,075 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-08T04:01:51,075 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-08T04:01:51,075 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T04:01:51,076 INFO [RS:0;32df4b4bb439:46837 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-08T04:01:51,076 INFO [RS:2;32df4b4bb439:39919 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-08T04:01:51,076 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-08T04:01:51,076 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-08T04:01:51,086 DEBUG [RS:2;32df4b4bb439:39919 {}] bucket.BucketCache(1487): Thread RS:2;32df4b4bb439:39919 started persisting bucket cache to file 2024-12-08T04:01:51,086 DEBUG [RS:0;32df4b4bb439:46837 {}] bucket.BucketCache(1487): Thread RS:0;32df4b4bb439:46837 started persisting bucket cache to file 2024-12-08T04:01:51,086 DEBUG [RS:0;32df4b4bb439:46837 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T04:01:51,086 DEBUG [RS:2;32df4b4bb439:39919 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-08T04:01:51,086 DEBUG [RS:0;32df4b4bb439:46837 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-08T04:01:51,086 DEBUG [RS:2;32df4b4bb439:39919 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-08T04:01:51,089 DEBUG [RS:0;32df4b4bb439:46837 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-08T04:01:51,089 DEBUG [RS:2;32df4b4bb439:39919 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-08T04:01:51,089 DEBUG [RS:0;32df4b4bb439:46837 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733630511086 2024-12-08T04:01:51,089 DEBUG [RS:2;32df4b4bb439:39919 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733630511086 2024-12-08T04:01:51,089 DEBUG [RS:2;32df4b4bb439:39919 {}] bucket.BucketCache(1509): Thread RS:2;32df4b4bb439:39919 finished persisting bucket cache to file, renaming 2024-12-08T04:01:51,089 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T04:01:51,089 INFO [RS:2;32df4b4bb439:39919 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T04:01:51,089 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T04:01:51,090 INFO [RS:2;32df4b4bb439:39919 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T04:01:51,090 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.HRegionServer(959): stopping server 32df4b4bb439,39919,1733630510408 2024-12-08T04:01:51,090 INFO [RS:2;32df4b4bb439:39919 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T04:01:51,090 INFO [RS:2;32df4b4bb439:39919 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;32df4b4bb439:39919. 2024-12-08T04:01:51,090 DEBUG [RS:2;32df4b4bb439:39919 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T04:01:51,090 DEBUG [RS:2;32df4b4bb439:39919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:51,090 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.HRegionServer(976): stopping server 32df4b4bb439,39919,1733630510408; all regions closed. 2024-12-08T04:01:51,091 DEBUG [RS:2;32df4b4bb439:39919 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/oldWALs 2024-12-08T04:01:51,091 INFO [RS:2;32df4b4bb439:39919 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32df4b4bb439%2C39919%2C1733630510408:(num 1733630510622) 2024-12-08T04:01:51,092 DEBUG [RS:2;32df4b4bb439:39919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:51,092 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T04:01:51,092 INFO [RS:2;32df4b4bb439:39919 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T04:01:51,092 INFO [RS:2;32df4b4bb439:39919 {}] hbase.ChoreService(370): Chore service for: regionserver/32df4b4bb439:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-08T04:01:51,092 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T04:01:51,092 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T04:01:51,092 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T04:01:51,092 INFO [regionserver/32df4b4bb439:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T04:01:51,092 INFO [RS:2;32df4b4bb439:39919 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T04:01:51,092 INFO [RS:2;32df4b4bb439:39919 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39919 2024-12-08T04:01:51,093 INFO [regionserver/32df4b4bb439:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T04:01:51,106 DEBUG [RS:0;32df4b4bb439:46837 {}] bucket.BucketCache(1509): Thread RS:0;32df4b4bb439:46837 finished persisting bucket cache to file, renaming 2024-12-08T04:01:51,106 WARN [RS:0;32df4b4bb439:46837 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-08T04:01:51,106 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T04:01:51,106 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T04:01:51,106 INFO [RS:0;32df4b4bb439:46837 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T04:01:51,106 INFO [RS:0;32df4b4bb439:46837 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T04:01:51,107 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer(3091): Received CLOSE for 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:51,107 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer(959): stopping server 32df4b4bb439,46837,1733630507033 2024-12-08T04:01:51,107 INFO [RS:0;32df4b4bb439:46837 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T04:01:51,107 INFO [RS:0;32df4b4bb439:46837 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;32df4b4bb439:46837. 2024-12-08T04:01:51,107 DEBUG [RS:0;32df4b4bb439:46837 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T04:01:51,107 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-08T04:01:51,107 DEBUG [RS:0;32df4b4bb439:46837 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:51,107 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 298cc6209906f23162b4dc1c414380a9, disabling compactions & flushes 2024-12-08T04:01:51,107 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:51,107 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T04:01:51,107 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T04:01:51,107 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:51,107 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T04:01:51,107 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. after waiting 0 ms 2024-12-08T04:01:51,107 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T04:01:51,107 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:51,108 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-08T04:01:51,108 DEBUG [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer(1325): Online Regions={298cc6209906f23162b4dc1c414380a9=testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9., 1588230740=hbase:meta,,1.1588230740} 2024-12-08T04:01:51,108 DEBUG [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 298cc6209906f23162b4dc1c414380a9 2024-12-08T04:01:51,108 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-08T04:01:51,108 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T04:01:51,108 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T04:01:51,108 DEBUG [StoreFileCloser-298cc6209906f23162b4dc1c414380a9-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/family/9b7c5317eb8442d3ac169926c6d671dc 2024-12-08T04:01:51,108 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T04:01:51,108 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T04:01:51,108 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T04:01:51,108 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-12-08T04:01:51,113 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/default/testBlockEvictionOnGracefulStop1733630508213/298cc6209906f23162b4dc1c414380a9/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-08T04:01:51,114 INFO [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:51,114 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 298cc6209906f23162b4dc1c414380a9: Waiting for close lock at 1733630511107Running coprocessor pre-close hooks at 1733630511107Disabling compacts and flushes for region at 1733630511107Disabling writes for close at 1733630511107Writing region close event to WAL at 1733630511108 (+1 ms)Running coprocessor post-close hooks at 1733630511114 (+6 ms)Closed at 1733630511114 2024-12-08T04:01:51,114 DEBUG [RS_CLOSE_REGION-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9. 2024-12-08T04:01:51,120 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7fe9465046b74846b98a57a9933de899 2024-12-08T04:01:51,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T04:01:51,133 INFO [RS:2;32df4b4bb439:39919 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T04:01:51,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a3f5bf0003, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/32df4b4bb439,39919,1733630510408 2024-12-08T04:01:51,135 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/info/7fe9465046b74846b98a57a9933de899 is 209, key is testBlockEvictionOnGracefulStop1733630508213,,1733630508218.298cc6209906f23162b4dc1c414380a9./info:regioninfo/1733630511072/Put/seqid=0 2024-12-08T04:01:51,136 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/info/7fe9465046b74846b98a57a9933de899 2024-12-08T04:01:51,138 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/.tmp/info/7fe9465046b74846b98a57a9933de899 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/info/7fe9465046b74846b98a57a9933de899 2024-12-08T04:01:51,139 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/info/7fe9465046b74846b98a57a9933de899, delay=961 ms 2024-12-08T04:01:51,140 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/info/7fe9465046b74846b98a57a9933de899, entries=8, sequenceid=19, filesize=6.6 K 2024-12-08T04:01:51,140 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=19, compaction requested=false 2024-12-08T04:01:51,141 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/info/14241f36394c430a8993febc097057ef 2024-12-08T04:01:51,141 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/info/7fe9465046b74846b98a57a9933de899 2024-12-08T04:01:51,142 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/ns/1c784d4fa2c345dc95e7056b8cc0023b 2024-12-08T04:01:51,142 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/table/c6877af379ba471bb8b891140f1dbd10 2024-12-08T04:01:51,145 INFO [regionserver/32df4b4bb439:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T04:01:51,145 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [32df4b4bb439,39919,1733630510408] 2024-12-08T04:01:51,146 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-12-08T04:01:51,146 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T04:01:51,146 INFO [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T04:01:51,146 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733630511108Running coprocessor pre-close hooks at 1733630511108Disabling compacts and flushes for region at 1733630511108Disabling writes for close at 1733630511108Obtaining lock to block concurrent updates at 1733630511108Preparing flush snapshotting stores in 1588230740 at 1733630511108Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1733630511109 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733630511109Flushing 1588230740/info: creating writer at 1733630511109Flushing 1588230740/info: appending metadata at 1733630511135 (+26 ms)Flushing 1588230740/info: closing flushed file at 1733630511135Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69ffeea7: reopening flushed file at 1733630511138 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=19, compaction requested=false at 1733630511140 (+2 ms)Writing region close event to WAL at 1733630511142 (+2 ms)Running coprocessor post-close hooks at 1733630511146 (+4 ms)Closed at 1733630511146 2024-12-08T04:01:51,146 DEBUG [RS_CLOSE_META-regionserver/32df4b4bb439:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T04:01:51,157 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/32df4b4bb439,39919,1733630510408 already deleted, retry=false 2024-12-08T04:01:51,157 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 32df4b4bb439,39919,1733630510408 expired; onlineServers=1 2024-12-08T04:01:51,180 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T04:01:51,180 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32df4b4bb439,46837,1733630507033, seqNum=-1] 2024-12-08T04:01:51,182 DEBUG [Async-Client-Retry-Timer-pool-0 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 298cc6209906f23162b4dc1c414380a9, server=32df4b4bb439,46837,1733630507033 because future has completed 2024-12-08T04:01:51,182 ERROR [Async-Client-Retry-Timer-pool-0 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$792/0x00007f4b30771938@3484c50c rejected from java.util.concurrent.ThreadPoolExecutor@1d557111[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 3] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T04:01:51,183 ERROR [Async-Client-Retry-Timer-pool-0 {}] assignment.RegionStateStore(295): FAILED persisting region=298cc6209906f23162b4dc1c414380a9 state=OPEN org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=32df4b4bb439:46837 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.mutate(ClientProtos.java:43836) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.lambda$mutate$0(RawAsyncTableImpl.java:180) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.call(ConnectionUtils.java:619) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.mutate(RawAsyncTableImpl.java:179) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.voidMutate(RawAsyncTableImpl.java:186) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.lambda$put$8(RawAsyncTableImpl.java:255) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 22 more 2024-12-08T04:01:51,184 ERROR [Async-Client-Retry-Timer-pool-0 {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-08T04:01:51,185 ERROR [Async-Client-Retry-Timer-pool-0 {}] master.HMaster(3223): ***** ABORTING master 32df4b4bb439,42935,1733630506800: FAILED persisting region=298cc6209906f23162b4dc1c414380a9 state=OPEN ***** org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=32df4b4bb439:46837 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.mutate(ClientProtos.java:43836) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.lambda$mutate$0(RawAsyncTableImpl.java:180) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.call(ConnectionUtils.java:619) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.mutate(RawAsyncTableImpl.java:179) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.voidMutate(RawAsyncTableImpl.java:186) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.lambda$put$8(RawAsyncTableImpl.java:255) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 22 more 2024-12-08T04:01:51,186 INFO [Async-Client-Retry-Timer-pool-0 {}] master.HMaster(3321): ***** STOPPING master '32df4b4bb439,42935,1733630506800' ***** 2024-12-08T04:01:51,186 INFO [Async-Client-Retry-Timer-pool-0 {}] master.HMaster(3323): STOPPED: Stopped by Async-Client-Retry-Timer-pool-0 2024-12-08T04:01:51,186 INFO [M:0;32df4b4bb439:42935 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T04:01:51,186 INFO [M:0;32df4b4bb439:42935 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T04:01:51,186 DEBUG [M:0;32df4b4bb439:42935 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-08T04:01:51,187 DEBUG [M:0;32df4b4bb439:42935 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-08T04:01:51,187 DEBUG [master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.small.0-1733630507441 {}] cleaner.HFileCleaner(306): Exit Thread[master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.small.0-1733630507441,5,FailOnTimeoutGroup] 2024-12-08T04:01:51,187 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-08T04:01:51,187 DEBUG [master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.large.0-1733630507440 {}] cleaner.HFileCleaner(306): Exit Thread[master/32df4b4bb439:0:becomeActiveMaster-HFileCleaner.large.0-1733630507440,5,FailOnTimeoutGroup] 2024-12-08T04:01:51,187 INFO [M:0;32df4b4bb439:42935 {}] hbase.ChoreService(370): Chore service for: master/32df4b4bb439:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-08T04:01:51,187 INFO [M:0;32df4b4bb439:42935 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T04:01:51,187 DEBUG [M:0;32df4b4bb439:42935 {}] master.HMaster(1795): Stopping service threads 2024-12-08T04:01:51,187 INFO [M:0;32df4b4bb439:42935 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-08T04:01:51,188 INFO [M:0;32df4b4bb439:42935 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T04:01:51,188 INFO [M:0;32df4b4bb439:42935 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-12-08T04:01:51,188 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-08T04:01:51,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-08T04:01:51,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-08T04:01:51,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:51,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T04:01:51,199 DEBUG [M:0;32df4b4bb439:42935 {}] zookeeper.ZKUtil(347): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-08T04:01:51,199 WARN [M:0;32df4b4bb439:42935 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-08T04:01:51,199 INFO [M:0;32df4b4bb439:42935 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/.lastflushedseqids 2024-12-08T04:01:51,199 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T04:01:51,202 INFO [M:0;32df4b4bb439:42935 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-08T04:01:51,202 INFO [M:0;32df4b4bb439:42935 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-12-08T04:01:51,203 DEBUG [M:0;32df4b4bb439:42935 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T04:01:51,203 INFO [M:0;32df4b4bb439:42935 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:51,203 DEBUG [M:0;32df4b4bb439:42935 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:51,203 DEBUG [M:0;32df4b4bb439:42935 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T04:01:51,203 DEBUG [M:0;32df4b4bb439:42935 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:51,204 ERROR [M:0;32df4b4bb439:42935 {}] regionserver.HRegion(1960): Memstore data size is 36887 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:51,204 INFO [M:0;32df4b4bb439:42935 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T04:01:51,204 DEBUG [M:0;32df4b4bb439:42935 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733630511203Disabling compacts and flushes for region at 1733630511203Disabling writes for close at 1733630511203Writing region close event to WAL at 1733630511204 (+1 ms)Closed at 1733630511204 2024-12-08T04:01:51,204 INFO [M:0;32df4b4bb439:42935 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-08T04:01:51,204 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T04:01:51,204 INFO [M:0;32df4b4bb439:42935 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42935 2024-12-08T04:01:51,205 INFO [M:0;32df4b4bb439:42935 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T04:01:51,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a3f5bf0003, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:51,246 INFO [RS:2;32df4b4bb439:39919 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T04:01:51,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39919-0x10003a3f5bf0003, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:51,246 INFO [RS:2;32df4b4bb439:39919 {}] regionserver.HRegionServer(1031): Exiting; stopping=32df4b4bb439,39919,1733630510408; zookeeper connection closed. 2024-12-08T04:01:51,246 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4a6da04f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4a6da04f 2024-12-08T04:01:51,308 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer(976): stopping server 32df4b4bb439,46837,1733630507033; all regions closed. 2024-12-08T04:01:51,312 DEBUG [RS:0;32df4b4bb439:46837 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/oldWALs 2024-12-08T04:01:51,312 INFO [RS:0;32df4b4bb439:46837 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32df4b4bb439%2C46837%2C1733630507033.meta:.meta(num 1733630510549) 2024-12-08T04:01:51,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:51,315 INFO [M:0;32df4b4bb439:42935 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T04:01:51,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42935-0x10003a3f5bf0000, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:51,316 DEBUG [RS:0;32df4b4bb439:46837 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da012785-0dbf-40a8-a8a9-1891431a41e2/84202073-64ab-e5f1-f747-631ef5868c68/oldWALs 2024-12-08T04:01:51,316 INFO [RS:0;32df4b4bb439:46837 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32df4b4bb439%2C46837%2C1733630507033:(num 1733630507783) 2024-12-08T04:01:51,316 DEBUG [RS:0;32df4b4bb439:46837 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T04:01:51,316 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T04:01:51,317 INFO [RS:0;32df4b4bb439:46837 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T04:01:51,317 INFO [RS:0;32df4b4bb439:46837 {}] hbase.ChoreService(370): Chore service for: regionserver/32df4b4bb439:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-08T04:01:51,317 INFO [RS:0;32df4b4bb439:46837 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T04:01:51,318 INFO [regionserver/32df4b4bb439:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T04:01:51,318 INFO [RS:0;32df4b4bb439:46837 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46837 2024-12-08T04:01:51,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/32df4b4bb439,46837,1733630507033 2024-12-08T04:01:51,332 INFO [RS:0;32df4b4bb439:46837 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T04:01:51,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:51,441 INFO [RS:0;32df4b4bb439:46837 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T04:01:51,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46837-0x10003a3f5bf0001, quorum=127.0.0.1:50055, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T04:01:51,441 INFO [RS:0;32df4b4bb439:46837 {}] regionserver.HRegionServer(1031): Exiting; stopping=32df4b4bb439,46837,1733630507033; zookeeper connection closed. 2024-12-08T04:01:51,441 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3b43cb65 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3b43cb65 2024-12-08T04:01:51,442 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-08T04:01:51,447 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-08T04:01:51,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-08T04:01:51,450 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T04:01:51,455 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 51) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=360 (was 382), ProcessCount=11 (was 11), AvailableMemoryMB=17389 (was 17511)