2024-11-28 07:44:29,593 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-28 07:44:29,610 main DEBUG Took 0.009434 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-28 07:44:29,610 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-28 07:44:29,611 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-28 07:44:29,612 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-28 07:44:29,613 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:44:29,619 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-28 07:44:29,630 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:44:29,632 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:44:29,632 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:44:29,633 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:44:29,633 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:44:29,633 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:44:29,634 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:44:29,635 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:44:29,635 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:44:29,635 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:44:29,636 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:44:29,636 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:44:29,637 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:44:29,637 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:44:29,637 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:44:29,638 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:44:29,638 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:44:29,638 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:44:29,639 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:44:29,639 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:44:29,640 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:44:29,640 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:44:29,640 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:44:29,641 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:44:29,641 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:44:29,642 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-28 07:44:29,644 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:44:29,645 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-28 07:44:29,647 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-28 07:44:29,648 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-28 07:44:29,649 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-28 07:44:29,649 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-28 07:44:29,657 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-28 07:44:29,659 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-28 07:44:29,661 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-28 07:44:29,661 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-28 07:44:29,661 main DEBUG createAppenders(={Console}) 2024-11-28 07:44:29,662 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-28 07:44:29,662 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-28 07:44:29,662 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-28 07:44:29,663 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-28 07:44:29,663 main DEBUG OutputStream closed 2024-11-28 07:44:29,663 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-28 07:44:29,664 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-28 07:44:29,664 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-28 07:44:29,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-28 07:44:29,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-28 07:44:29,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-28 07:44:29,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-28 07:44:29,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-28 07:44:29,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-28 07:44:29,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-28 07:44:29,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-28 07:44:29,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-28 07:44:29,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-28 07:44:29,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-28 07:44:29,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-28 07:44:29,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-28 07:44:29,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-28 07:44:29,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-28 07:44:29,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-28 07:44:29,735 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-28 07:44:29,735 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-28 07:44:29,738 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28 07:44:29,738 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-28 07:44:29,738 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-28 07:44:29,739 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-28T07:44:29,753 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-28 07:44:29,756 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-28 07:44:29,756 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28T07:44:29,974 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e 2024-11-28T07:44:29,995 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=299, ProcessCount=11, AvailableMemoryMB=7496 2024-11-28T07:44:30,081 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-28T07:44:30,170 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:30,186 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/cluster_28568c51-b565-e7f8-4a5a-c3b655501470, deleteOnExit=true 2024-11-28T07:44:30,281 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/cluster_28568c51-b565-e7f8-4a5a-c3b655501470/zookeeper_0, clientPort=50908, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/cluster_28568c51-b565-e7f8-4a5a-c3b655501470/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/cluster_28568c51-b565-e7f8-4a5a-c3b655501470/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-28T07:44:30,293 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50908 2024-11-28T07:44:30,294 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:30,389 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597 with version=8 2024-11-28T07:44:30,390 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/hbase-staging 2024-11-28T07:44:30,488 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-28T07:44:30,768 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f64cf8bdf19b:0 server-side Connection retries=45 2024-11-28T07:44:30,778 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:30,780 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:30,788 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T07:44:30,788 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:30,788 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T07:44:30,925 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-28T07:44:31,015 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T07:44:31,051 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-28T07:44:31,061 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-28T07:44:31,065 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T07:44:31,108 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 12514 (auto-detected) 2024-11-28T07:44:31,109 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-28T07:44:31,132 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43355 2024-11-28T07:44:31,154 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43355 connecting to ZooKeeper ensemble=127.0.0.1:50908 2024-11-28T07:44:31,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:433550x0, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T07:44:31,190 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43355-0x1015d8872c50000 connected 2024-11-28T07:44:31,224 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:31,225 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:31,233 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:44:31,237 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597, hbase.cluster.distributed=false 2024-11-28T07:44:31,259 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T07:44:31,264 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43355 2024-11-28T07:44:31,264 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43355 2024-11-28T07:44:31,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43355 2024-11-28T07:44:31,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43355 2024-11-28T07:44:31,266 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43355 2024-11-28T07:44:31,357 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f64cf8bdf19b:0 server-side Connection retries=45 2024-11-28T07:44:31,359 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:31,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:31,360 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T07:44:31,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:31,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T07:44:31,363 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T07:44:31,367 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T07:44:31,368 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36377 2024-11-28T07:44:31,370 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36377 connecting to ZooKeeper ensemble=127.0.0.1:50908 2024-11-28T07:44:31,371 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:31,372 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:31,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:363770x0, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T07:44:31,377 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36377-0x1015d8872c50001 connected 2024-11-28T07:44:31,377 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:44:31,383 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T07:44:31,397 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/bucket.cache 2024-11-28T07:44:31,404 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-28T07:44:31,405 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-28T07:44:31,407 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-28T07:44:31,409 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-28T07:44:31,409 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-28T07:44:31,415 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-28T07:44:31,417 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-28T07:44:31,418 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/bucket.persistence is complete. 2024-11-28T07:44:31,421 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T07:44:31,423 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T07:44:31,428 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T07:44:31,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36377 2024-11-28T07:44:31,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36377 2024-11-28T07:44:31,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36377 2024-11-28T07:44:31,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36377 2024-11-28T07:44:31,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36377 2024-11-28T07:44:31,460 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f64cf8bdf19b:0 server-side Connection retries=45 2024-11-28T07:44:31,460 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:31,461 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:31,461 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T07:44:31,461 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:31,462 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T07:44:31,462 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T07:44:31,462 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T07:44:31,464 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34747 2024-11-28T07:44:31,466 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34747 connecting to ZooKeeper ensemble=127.0.0.1:50908 2024-11-28T07:44:31,467 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:31,468 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:31,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:347470x0, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T07:44:31,474 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34747-0x1015d8872c50002 connected 2024-11-28T07:44:31,474 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:44:31,475 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T07:44:31,480 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/bucket.cache 2024-11-28T07:44:31,481 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-28T07:44:31,481 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-28T07:44:31,481 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-28T07:44:31,482 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-28T07:44:31,482 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-28T07:44:31,482 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-28T07:44:31,483 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-28T07:44:31,483 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/bucket.persistence is complete. 2024-11-28T07:44:31,485 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T07:44:31,486 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T07:44:31,488 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T07:44:31,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34747 2024-11-28T07:44:31,493 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34747 2024-11-28T07:44:31,495 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34747 2024-11-28T07:44:31,495 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34747 2024-11-28T07:44:31,496 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34747 2024-11-28T07:44:31,513 DEBUG [M:0;f64cf8bdf19b:43355 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f64cf8bdf19b:43355 2024-11-28T07:44:31,515 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f64cf8bdf19b,43355,1732779870557 2024-11-28T07:44:31,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:44:31,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:44:31,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:44:31,527 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f64cf8bdf19b,43355,1732779870557 2024-11-28T07:44:31,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T07:44:31,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T07:44:31,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:31,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:31,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:31,553 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T07:44:31,555 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f64cf8bdf19b,43355,1732779870557 from backup master directory 2024-11-28T07:44:31,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:44:31,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:44:31,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f64cf8bdf19b,43355,1732779870557 2024-11-28T07:44:31,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:44:31,559 WARN [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T07:44:31,559 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f64cf8bdf19b,43355,1732779870557 2024-11-28T07:44:31,562 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-28T07:44:31,563 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-28T07:44:31,571 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/hbase.id] with ID: cb2b68d6-e068-46b7-b262-7c1bf425ee5a 2024-11-28T07:44:31,571 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/.tmp/hbase.id 2024-11-28T07:44:31,578 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/hbase.id] 2024-11-28T07:44:31,617 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:31,623 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-28T07:44:31,643 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-28T07:44:31,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:31,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:31,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:31,675 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T07:44:31,679 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-28T07:44:31,730 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:44:31,739 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:44:31,807 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store 2024-11-28T07:44:31,832 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-28T07:44:31,837 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:44:31,838 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T07:44:31,838 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:44:31,839 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:44:31,841 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T07:44:31,841 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:44:31,841 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:44:31,842 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732779871838Disabling compacts and flushes for region at 1732779871838Disabling writes for close at 1732779871841 (+3 ms)Writing region close event to WAL at 1732779871841Closed at 1732779871841 2024-11-28T07:44:31,843 WARN [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/.initializing 2024-11-28T07:44:31,843 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/WALs/f64cf8bdf19b,43355,1732779870557 2024-11-28T07:44:31,857 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T07:44:31,872 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f64cf8bdf19b%2C43355%2C1732779870557, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/WALs/f64cf8bdf19b,43355,1732779870557, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/oldWALs, maxLogs=10 2024-11-28T07:44:31,914 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/WALs/f64cf8bdf19b,43355,1732779870557/f64cf8bdf19b%2C43355%2C1732779870557.1732779871878 2024-11-28T07:44:31,915 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T07:44:31,915 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:44:31,916 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:44:31,919 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:31,920 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:31,958 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:31,987 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-28T07:44:31,991 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:31,993 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:31,994 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:32,000 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-28T07:44:32,000 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:32,000 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:44:32,001 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:32,006 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-28T07:44:32,006 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:32,006 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:44:32,006 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:32,011 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-28T07:44:32,011 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:32,012 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:44:32,012 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:32,016 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:32,017 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:32,020 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:32,021 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:32,023 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T07:44:32,027 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:32,035 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:44:32,036 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59942565, jitterRate=-0.10678617656230927}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T07:44:32,042 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732779871933Initializing all the Stores at 1732779871933Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779871934 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779871935 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779871935Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779871935Cleaning up temporary data from old regions at 1732779872021 (+86 ms)Region opened successfully at 1732779872042 (+21 ms) 2024-11-28T07:44:32,044 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-28T07:44:32,077 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16f9dad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f64cf8bdf19b/172.17.0.2:0 2024-11-28T07:44:32,111 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-28T07:44:32,128 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-28T07:44:32,129 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-28T07:44:32,133 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:44:32,134 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:44:32,142 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-28T07:44:32,142 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-28T07:44:32,176 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-28T07:44:32,186 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-28T07:44:32,189 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-28T07:44:32,192 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-28T07:44:32,194 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-28T07:44:32,195 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-28T07:44:32,197 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-28T07:44:32,201 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-28T07:44:32,203 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-28T07:44:32,204 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-28T07:44:32,205 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-28T07:44:32,225 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-28T07:44:32,227 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-28T07:44:32,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T07:44:32,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T07:44:32,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:32,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T07:44:32,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:32,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:32,246 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f64cf8bdf19b,43355,1732779870557, sessionid=0x1015d8872c50000, setting cluster-up flag (Was=false) 2024-11-28T07:44:32,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:32,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:32,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:32,266 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-28T07:44:32,268 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f64cf8bdf19b,43355,1732779870557 2024-11-28T07:44:32,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:32,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:32,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:32,280 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-28T07:44:32,281 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f64cf8bdf19b,43355,1732779870557 2024-11-28T07:44:32,287 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-28T07:44:32,300 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(746): ClusterId : cb2b68d6-e068-46b7-b262-7c1bf425ee5a 2024-11-28T07:44:32,300 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(746): ClusterId : cb2b68d6-e068-46b7-b262-7c1bf425ee5a 2024-11-28T07:44:32,303 DEBUG [RS:1;f64cf8bdf19b:34747 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T07:44:32,303 DEBUG [RS:0;f64cf8bdf19b:36377 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T07:44:32,309 DEBUG [RS:1;f64cf8bdf19b:34747 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T07:44:32,309 DEBUG [RS:0;f64cf8bdf19b:36377 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T07:44:32,309 DEBUG [RS:1;f64cf8bdf19b:34747 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T07:44:32,309 DEBUG [RS:0;f64cf8bdf19b:36377 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T07:44:32,314 DEBUG [RS:0;f64cf8bdf19b:36377 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T07:44:32,314 DEBUG [RS:1;f64cf8bdf19b:34747 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T07:44:32,314 DEBUG [RS:0;f64cf8bdf19b:36377 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ffff872, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f64cf8bdf19b/172.17.0.2:0 2024-11-28T07:44:32,314 DEBUG [RS:1;f64cf8bdf19b:34747 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@246f6ad4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f64cf8bdf19b/172.17.0.2:0 2024-11-28T07:44:32,338 DEBUG [RS:0;f64cf8bdf19b:36377 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f64cf8bdf19b:36377 2024-11-28T07:44:32,338 DEBUG [RS:1;f64cf8bdf19b:34747 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f64cf8bdf19b:34747 2024-11-28T07:44:32,342 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T07:44:32,342 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T07:44:32,342 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T07:44:32,342 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T07:44:32,342 DEBUG [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T07:44:32,342 DEBUG [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T07:44:32,345 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(2659): reportForDuty to master=f64cf8bdf19b,43355,1732779870557 with port=34747, startcode=1732779871459 2024-11-28T07:44:32,345 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(2659): reportForDuty to master=f64cf8bdf19b,43355,1732779870557 with port=36377, startcode=1732779871317 2024-11-28T07:44:32,358 DEBUG [RS:0;f64cf8bdf19b:36377 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T07:44:32,358 DEBUG [RS:1;f64cf8bdf19b:34747 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T07:44:32,382 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:44:32,394 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-28T07:44:32,404 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-28T07:44:32,412 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f64cf8bdf19b,43355,1732779870557 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-28T07:44:32,423 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f64cf8bdf19b:0, corePoolSize=5, maxPoolSize=5 2024-11-28T07:44:32,423 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f64cf8bdf19b:0, corePoolSize=5, maxPoolSize=5 2024-11-28T07:44:32,423 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f64cf8bdf19b:0, corePoolSize=5, maxPoolSize=5 2024-11-28T07:44:32,424 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f64cf8bdf19b:0, corePoolSize=5, maxPoolSize=5 2024-11-28T07:44:32,424 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f64cf8bdf19b:0, corePoolSize=10, maxPoolSize=10 2024-11-28T07:44:32,424 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,424 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f64cf8bdf19b:0, corePoolSize=2, maxPoolSize=2 2024-11-28T07:44:32,425 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,434 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732779902433 2024-11-28T07:44:32,437 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-28T07:44:32,437 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36715, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T07:44:32,437 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58585, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T07:44:32,438 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T07:44:32,439 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-28T07:44:32,439 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-28T07:44:32,443 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-28T07:44:32,443 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-28T07:44:32,444 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-28T07:44:32,444 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-28T07:44:32,445 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:32,445 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-28T07:44:32,446 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43355 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:44:32,446 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,451 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-28T07:44:32,453 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-28T07:44:32,454 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-28T07:44:32,454 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43355 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:44:32,456 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-28T07:44:32,456 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-28T07:44:32,459 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.large.0-1732779872459,5,FailOnTimeoutGroup] 2024-11-28T07:44:32,460 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.small.0-1732779872459,5,FailOnTimeoutGroup] 2024-11-28T07:44:32,460 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,460 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-28T07:44:32,461 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,462 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,465 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-28T07:44:32,466 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597 2024-11-28T07:44:32,479 DEBUG [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-28T07:44:32,479 DEBUG [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-28T07:44:32,479 WARN [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-28T07:44:32,479 WARN [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-28T07:44:32,480 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:44:32,481 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-28T07:44:32,485 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T07:44:32,486 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:32,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:32,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-28T07:44:32,490 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T07:44:32,490 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:32,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:32,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-28T07:44:32,494 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T07:44:32,494 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:32,494 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:32,495 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-28T07:44:32,498 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T07:44:32,498 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:32,499 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:32,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T07:44:32,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740 2024-11-28T07:44:32,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740 2024-11-28T07:44:32,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T07:44:32,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T07:44:32,502 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T07:44:32,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T07:44:32,509 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:44:32,510 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61083864, jitterRate=-0.08977949619293213}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T07:44:32,513 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732779872481Initializing all the Stores at 1732779872481Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779872481Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779872481Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779872481Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779872481Cleaning up temporary data from old regions at 1732779872501 (+20 ms)Region opened successfully at 1732779872513 (+12 ms) 2024-11-28T07:44:32,513 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T07:44:32,513 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T07:44:32,513 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T07:44:32,514 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T07:44:32,514 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T07:44:32,515 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T07:44:32,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732779872513Disabling compacts and flushes for region at 1732779872513Disabling writes for close at 1732779872514 (+1 ms)Writing region close event to WAL at 1732779872515 (+1 ms)Closed at 1732779872515 2024-11-28T07:44:32,520 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T07:44:32,520 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-28T07:44:32,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T07:44:32,535 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:44:32,537 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-28T07:44:32,581 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(2659): reportForDuty to master=f64cf8bdf19b,43355,1732779870557 with port=34747, startcode=1732779871459 2024-11-28T07:44:32,581 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(2659): reportForDuty to master=f64cf8bdf19b,43355,1732779870557 with port=36377, startcode=1732779871317 2024-11-28T07:44:32,583 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43355 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f64cf8bdf19b,34747,1732779871459 2024-11-28T07:44:32,586 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43355 {}] master.ServerManager(517): Registering regionserver=f64cf8bdf19b,34747,1732779871459 2024-11-28T07:44:32,594 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43355 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f64cf8bdf19b,36377,1732779871317 2024-11-28T07:44:32,594 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43355 {}] master.ServerManager(517): Registering regionserver=f64cf8bdf19b,36377,1732779871317 2024-11-28T07:44:32,595 DEBUG [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597 2024-11-28T07:44:32,595 DEBUG [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-28T07:44:32,595 DEBUG [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T07:44:32,596 DEBUG [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597 2024-11-28T07:44:32,597 DEBUG [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-28T07:44:32,597 DEBUG [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T07:44:32,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T07:44:32,607 DEBUG [RS:1;f64cf8bdf19b:34747 {}] zookeeper.ZKUtil(111): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f64cf8bdf19b,34747,1732779871459 2024-11-28T07:44:32,607 WARN [RS:1;f64cf8bdf19b:34747 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T07:44:32,607 INFO [RS:1;f64cf8bdf19b:34747 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:44:32,607 DEBUG [RS:0;f64cf8bdf19b:36377 {}] zookeeper.ZKUtil(111): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f64cf8bdf19b,36377,1732779871317 2024-11-28T07:44:32,608 WARN [RS:0;f64cf8bdf19b:36377 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T07:44:32,608 DEBUG [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,34747,1732779871459 2024-11-28T07:44:32,608 INFO [RS:0;f64cf8bdf19b:36377 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:44:32,608 DEBUG [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,36377,1732779871317 2024-11-28T07:44:32,610 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f64cf8bdf19b,34747,1732779871459] 2024-11-28T07:44:32,610 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f64cf8bdf19b,36377,1732779871317] 2024-11-28T07:44:32,638 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T07:44:32,638 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T07:44:32,654 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T07:44:32,654 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T07:44:32,659 INFO [RS:0;f64cf8bdf19b:36377 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T07:44:32,659 INFO [RS:1;f64cf8bdf19b:34747 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T07:44:32,659 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,659 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,660 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T07:44:32,660 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T07:44:32,665 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T07:44:32,665 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T07:44:32,667 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,667 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,667 DEBUG [RS:0;f64cf8bdf19b:36377 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,667 DEBUG [RS:1;f64cf8bdf19b:34747 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,667 DEBUG [RS:1;f64cf8bdf19b:34747 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,668 DEBUG [RS:1;f64cf8bdf19b:34747 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,668 DEBUG [RS:1;f64cf8bdf19b:34747 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,668 DEBUG [RS:0;f64cf8bdf19b:36377 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,668 DEBUG [RS:0;f64cf8bdf19b:36377 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,668 DEBUG [RS:1;f64cf8bdf19b:34747 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,668 DEBUG [RS:0;f64cf8bdf19b:36377 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,668 DEBUG [RS:1;f64cf8bdf19b:34747 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f64cf8bdf19b:0, corePoolSize=2, maxPoolSize=2 2024-11-28T07:44:32,669 DEBUG [RS:0;f64cf8bdf19b:36377 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,669 DEBUG [RS:1;f64cf8bdf19b:34747 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,669 DEBUG [RS:0;f64cf8bdf19b:36377 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f64cf8bdf19b:0, corePoolSize=2, maxPoolSize=2 2024-11-28T07:44:32,669 DEBUG [RS:1;f64cf8bdf19b:34747 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,669 DEBUG [RS:0;f64cf8bdf19b:36377 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,669 DEBUG [RS:1;f64cf8bdf19b:34747 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,669 DEBUG [RS:0;f64cf8bdf19b:36377 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,669 DEBUG [RS:1;f64cf8bdf19b:34747 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,669 DEBUG [RS:0;f64cf8bdf19b:36377 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,669 DEBUG [RS:1;f64cf8bdf19b:34747 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,669 DEBUG [RS:0;f64cf8bdf19b:36377 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,669 DEBUG [RS:1;f64cf8bdf19b:34747 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,670 DEBUG [RS:0;f64cf8bdf19b:36377 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,670 DEBUG [RS:1;f64cf8bdf19b:34747 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f64cf8bdf19b:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:44:32,670 DEBUG [RS:0;f64cf8bdf19b:36377 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:32,670 DEBUG [RS:1;f64cf8bdf19b:34747 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:44:32,670 DEBUG [RS:0;f64cf8bdf19b:36377 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f64cf8bdf19b:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:44:32,670 DEBUG [RS:0;f64cf8bdf19b:36377 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:44:32,672 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,673 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,673 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,673 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,673 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,673 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,36377,1732779871317-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T07:44:32,675 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,675 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,675 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,675 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,675 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,675 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,34747,1732779871459-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T07:44:32,688 WARN [f64cf8bdf19b:43355 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T07:44:32,703 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T07:44:32,703 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T07:44:32,705 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,36377,1732779871317-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,705 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,34747,1732779871459-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,705 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,705 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,705 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.Replication(171): f64cf8bdf19b,36377,1732779871317 started 2024-11-28T07:44:32,705 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.Replication(171): f64cf8bdf19b,34747,1732779871459 started 2024-11-28T07:44:32,725 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,726 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(1482): Serving as f64cf8bdf19b,34747,1732779871459, RpcServer on f64cf8bdf19b/172.17.0.2:34747, sessionid=0x1015d8872c50002 2024-11-28T07:44:32,727 DEBUG [RS:1;f64cf8bdf19b:34747 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T07:44:32,727 DEBUG [RS:1;f64cf8bdf19b:34747 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f64cf8bdf19b,34747,1732779871459 2024-11-28T07:44:32,727 DEBUG [RS:1;f64cf8bdf19b:34747 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f64cf8bdf19b,34747,1732779871459' 2024-11-28T07:44:32,727 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:32,727 DEBUG [RS:1;f64cf8bdf19b:34747 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T07:44:32,728 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(1482): Serving as f64cf8bdf19b,36377,1732779871317, RpcServer on f64cf8bdf19b/172.17.0.2:36377, sessionid=0x1015d8872c50001 2024-11-28T07:44:32,728 DEBUG [RS:0;f64cf8bdf19b:36377 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T07:44:32,728 DEBUG [RS:0;f64cf8bdf19b:36377 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f64cf8bdf19b,36377,1732779871317 2024-11-28T07:44:32,728 DEBUG [RS:0;f64cf8bdf19b:36377 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f64cf8bdf19b,36377,1732779871317' 2024-11-28T07:44:32,728 DEBUG [RS:0;f64cf8bdf19b:36377 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T07:44:32,729 DEBUG [RS:1;f64cf8bdf19b:34747 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T07:44:32,729 DEBUG [RS:0;f64cf8bdf19b:36377 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T07:44:32,729 DEBUG [RS:1;f64cf8bdf19b:34747 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T07:44:32,730 DEBUG [RS:1;f64cf8bdf19b:34747 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T07:44:32,730 DEBUG [RS:1;f64cf8bdf19b:34747 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f64cf8bdf19b,34747,1732779871459 2024-11-28T07:44:32,730 DEBUG [RS:1;f64cf8bdf19b:34747 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f64cf8bdf19b,34747,1732779871459' 2024-11-28T07:44:32,730 DEBUG [RS:1;f64cf8bdf19b:34747 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T07:44:32,730 DEBUG [RS:0;f64cf8bdf19b:36377 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T07:44:32,730 DEBUG [RS:0;f64cf8bdf19b:36377 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T07:44:32,730 DEBUG [RS:0;f64cf8bdf19b:36377 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f64cf8bdf19b,36377,1732779871317 2024-11-28T07:44:32,730 DEBUG [RS:0;f64cf8bdf19b:36377 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f64cf8bdf19b,36377,1732779871317' 2024-11-28T07:44:32,730 DEBUG [RS:0;f64cf8bdf19b:36377 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T07:44:32,731 DEBUG [RS:1;f64cf8bdf19b:34747 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T07:44:32,731 DEBUG [RS:0;f64cf8bdf19b:36377 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T07:44:32,731 DEBUG [RS:1;f64cf8bdf19b:34747 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T07:44:32,732 INFO [RS:1;f64cf8bdf19b:34747 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T07:44:32,732 INFO [RS:1;f64cf8bdf19b:34747 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T07:44:32,732 DEBUG [RS:0;f64cf8bdf19b:36377 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T07:44:32,732 INFO [RS:0;f64cf8bdf19b:36377 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T07:44:32,732 INFO [RS:0;f64cf8bdf19b:36377 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T07:44:32,837 INFO [RS:0;f64cf8bdf19b:36377 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T07:44:32,838 INFO [RS:1;f64cf8bdf19b:34747 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T07:44:32,839 INFO [RS:0;f64cf8bdf19b:36377 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f64cf8bdf19b%2C36377%2C1732779871317, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,36377,1732779871317, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/oldWALs, maxLogs=55 2024-11-28T07:44:32,839 INFO [RS:1;f64cf8bdf19b:34747 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f64cf8bdf19b%2C34747%2C1732779871459, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,34747,1732779871459, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/oldWALs, maxLogs=55 2024-11-28T07:44:32,865 INFO [RS:1;f64cf8bdf19b:34747 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,34747,1732779871459/f64cf8bdf19b%2C34747%2C1732779871459.1732779872846 2024-11-28T07:44:32,865 INFO [RS:0;f64cf8bdf19b:36377 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,36377,1732779871317/f64cf8bdf19b%2C36377%2C1732779871317.1732779872846 2024-11-28T07:44:32,866 DEBUG [RS:1;f64cf8bdf19b:34747 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T07:44:32,866 DEBUG [RS:0;f64cf8bdf19b:36377 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T07:44:32,940 DEBUG [f64cf8bdf19b:43355 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-28T07:44:32,952 DEBUG [f64cf8bdf19b:43355 {}] balancer.BalancerClusterState(204): Hosts are {f64cf8bdf19b=0} racks are {/default-rack=0} 2024-11-28T07:44:32,955 DEBUG [f64cf8bdf19b:43355 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T07:44:32,955 DEBUG [f64cf8bdf19b:43355 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T07:44:32,955 DEBUG [f64cf8bdf19b:43355 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T07:44:32,955 DEBUG [f64cf8bdf19b:43355 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T07:44:32,955 INFO [f64cf8bdf19b:43355 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T07:44:32,955 INFO [f64cf8bdf19b:43355 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T07:44:32,956 DEBUG [f64cf8bdf19b:43355 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T07:44:32,963 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f64cf8bdf19b,36377,1732779871317 2024-11-28T07:44:32,968 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f64cf8bdf19b,36377,1732779871317, state=OPENING 2024-11-28T07:44:32,973 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-28T07:44:32,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:32,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:32,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:32,976 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:32,976 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:32,977 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:32,977 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:44:32,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f64cf8bdf19b,36377,1732779871317}] 2024-11-28T07:44:33,152 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T07:44:33,155 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51905, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T07:44:33,163 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T07:44:33,163 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:44:33,164 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T07:44:33,165 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f64cf8bdf19b%2C36377%2C1732779871317.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,36377,1732779871317, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/oldWALs, maxLogs=55 2024-11-28T07:44:33,187 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,36377,1732779871317/f64cf8bdf19b%2C36377%2C1732779871317.meta.1732779873167.meta 2024-11-28T07:44:33,187 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T07:44:33,188 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:44:33,189 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T07:44:33,192 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T07:44:33,197 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T07:44:33,201 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T07:44:33,202 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:44:33,202 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T07:44:33,202 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T07:44:33,203 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-28T07:44:33,205 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T07:44:33,205 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:33,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:33,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-28T07:44:33,206 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T07:44:33,206 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:33,207 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:33,207 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-28T07:44:33,207 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T07:44:33,208 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:33,208 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:33,208 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-28T07:44:33,209 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T07:44:33,209 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:33,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:33,209 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T07:44:33,210 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740 2024-11-28T07:44:33,211 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740 2024-11-28T07:44:33,211 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T07:44:33,211 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T07:44:33,212 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T07:44:33,213 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T07:44:33,214 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59818831, jitterRate=-0.10862995684146881}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T07:44:33,214 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T07:44:33,215 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732779873202Writing region info on filesystem at 1732779873203 (+1 ms)Initializing all the Stores at 1732779873203Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779873203Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779873203Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779873203Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779873203Cleaning up temporary data from old regions at 1732779873212 (+9 ms)Running coprocessor post-open hooks at 1732779873214 (+2 ms)Region opened successfully at 1732779873215 (+1 ms) 2024-11-28T07:44:33,221 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732779873143 2024-11-28T07:44:33,233 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T07:44:33,233 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T07:44:33,235 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f64cf8bdf19b,36377,1732779871317 2024-11-28T07:44:33,236 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f64cf8bdf19b,36377,1732779871317, state=OPEN 2024-11-28T07:44:33,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:44:33,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:44:33,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:44:33,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:33,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:33,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:33,240 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f64cf8bdf19b,36377,1732779871317 2024-11-28T07:44:33,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-28T07:44:33,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f64cf8bdf19b,36377,1732779871317 in 261 msec 2024-11-28T07:44:33,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-28T07:44:33,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 718 msec 2024-11-28T07:44:33,248 DEBUG [PEWorker-4 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T07:44:33,248 INFO [PEWorker-4 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-28T07:44:33,270 DEBUG [PEWorker-4 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:44:33,272 DEBUG [PEWorker-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,36377,1732779871317, seqNum=-1] 2024-11-28T07:44:33,293 DEBUG [PEWorker-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:44:33,295 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43039, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:44:33,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 998 msec 2024-11-28T07:44:33,322 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732779873322, completionTime=-1 2024-11-28T07:44:33,325 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-28T07:44:33,326 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:44:33,351 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:44:33,351 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732779933351 2024-11-28T07:44:33,351 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732779993351 2024-11-28T07:44:33,351 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-28T07:44:33,353 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-28T07:44:33,360 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,43355,1732779870557-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:33,360 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,43355,1732779870557-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:33,360 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,43355,1732779870557-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:33,362 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f64cf8bdf19b:43355, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:33,362 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:33,363 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:33,368 DEBUG [master/f64cf8bdf19b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-28T07:44:33,391 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.831sec 2024-11-28T07:44:33,393 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-28T07:44:33,394 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-28T07:44:33,395 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-28T07:44:33,396 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-28T07:44:33,396 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-28T07:44:33,397 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,43355,1732779870557-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T07:44:33,398 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,43355,1732779870557-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-28T07:44:33,399 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-28T07:44:33,400 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-28T07:44:33,401 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,43355,1732779870557-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:33,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8eba43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:44:33,412 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-28T07:44:33,412 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-28T07:44:33,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,43355,-1 for getting cluster id 2024-11-28T07:44:33,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:44:33,426 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb2b68d6-e068-46b7-b262-7c1bf425ee5a' 2024-11-28T07:44:33,429 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:44:33,429 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb2b68d6-e068-46b7-b262-7c1bf425ee5a" 2024-11-28T07:44:33,430 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a86afb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:44:33,430 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,43355,-1] 2024-11-28T07:44:33,433 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:44:33,435 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:33,436 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:44:33,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fe5747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:44:33,439 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:44:33,446 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,36377,1732779871317, seqNum=-1] 2024-11-28T07:44:33,446 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:44:33,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:44:33,484 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f64cf8bdf19b,43355,1732779870557 2024-11-28T07:44:33,491 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T07:44:33,496 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f64cf8bdf19b,43355,1732779870557 2024-11-28T07:44:33,498 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@303293d6 2024-11-28T07:44:33,499 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T07:44:33,502 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T07:44:33,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732779873484', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T07:44:33,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732779873484 2024-11-28T07:44:33,515 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732779873484 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T07:44:33,517 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732779873484" procId is: 4 2024-11-28T07:44:33,518 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:33,519 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732779873484 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T07:44:33,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:44:33,533 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732779873484-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1a4894ccf73e6e061b201e0caa5d0423, NAME => 'testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732779873484', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597 2024-11-28T07:44:33,539 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732779873484-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:44:33,540 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732779873484-pool-0 {}] regionserver.HRegion(1722): Closing 1a4894ccf73e6e061b201e0caa5d0423, disabling compactions & flushes 2024-11-28T07:44:33,540 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732779873484-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:33,540 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732779873484-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:33,540 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732779873484-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. after waiting 0 ms 2024-11-28T07:44:33,540 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732779873484-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:33,540 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732779873484-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:33,540 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732779873484-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1a4894ccf73e6e061b201e0caa5d0423: Waiting for close lock at 1732779873540Disabling compacts and flushes for region at 1732779873540Disabling writes for close at 1732779873540Writing region close event to WAL at 1732779873540Closed at 1732779873540 2024-11-28T07:44:33,541 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732779873484 execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T07:44:33,546 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732779873542"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732779873542"}]},"ts":"1732779873542"} 2024-11-28T07:44:33,550 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T07:44:33,551 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732779873484 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T07:44:33,554 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732779873484","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779873551"}]},"ts":"1732779873551"} 2024-11-28T07:44:33,558 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732779873484, state=ENABLING in hbase:meta 2024-11-28T07:44:33,558 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(204): Hosts are {f64cf8bdf19b=0} racks are {/default-rack=0} 2024-11-28T07:44:33,559 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T07:44:33,559 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T07:44:33,559 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T07:44:33,559 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T07:44:33,559 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T07:44:33,559 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T07:44:33,559 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T07:44:33,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732779873484, region=1a4894ccf73e6e061b201e0caa5d0423, ASSIGN}] 2024-11-28T07:44:33,561 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732779873484, region=1a4894ccf73e6e061b201e0caa5d0423, ASSIGN 2024-11-28T07:44:33,562 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732779873484, region=1a4894ccf73e6e061b201e0caa5d0423, ASSIGN; state=OFFLINE, location=f64cf8bdf19b,36377,1732779871317; forceNewPlan=false, retain=false 2024-11-28T07:44:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:44:33,715 INFO [f64cf8bdf19b:43355 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-28T07:44:33,716 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1a4894ccf73e6e061b201e0caa5d0423, regionState=OPENING, regionLocation=f64cf8bdf19b,36377,1732779871317 2024-11-28T07:44:33,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732779873484, region=1a4894ccf73e6e061b201e0caa5d0423, ASSIGN because future has completed 2024-11-28T07:44:33,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a4894ccf73e6e061b201e0caa5d0423, server=f64cf8bdf19b,36377,1732779871317}] 2024-11-28T07:44:33,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:44:33,876 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:33,876 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a4894ccf73e6e061b201e0caa5d0423, NAME => 'testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:44:33,877 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732779873484 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:33,877 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:44:33,877 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:33,877 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:33,878 INFO [StoreOpener-1a4894ccf73e6e061b201e0caa5d0423-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:33,883 INFO [StoreOpener-1a4894ccf73e6e061b201e0caa5d0423-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a4894ccf73e6e061b201e0caa5d0423 columnFamilyName family 2024-11-28T07:44:33,883 DEBUG [StoreOpener-1a4894ccf73e6e061b201e0caa5d0423-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:33,883 INFO [StoreOpener-1a4894ccf73e6e061b201e0caa5d0423-1 {}] regionserver.HStore(327): Store=1a4894ccf73e6e061b201e0caa5d0423/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:44:33,883 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:33,884 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:33,884 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:33,885 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:33,885 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:33,886 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:33,892 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:44:33,892 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1a4894ccf73e6e061b201e0caa5d0423; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59624356, jitterRate=-0.11152786016464233}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:44:33,893 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:33,893 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1a4894ccf73e6e061b201e0caa5d0423: Running coprocessor pre-open hook at 1732779873877Writing region info on filesystem at 1732779873877Initializing all the Stores at 1732779873877Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779873877Cleaning up temporary data from old regions at 1732779873885 (+8 ms)Running coprocessor post-open hooks at 1732779873893 (+8 ms)Region opened successfully at 1732779873893 2024-11-28T07:44:33,894 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423., pid=6, masterSystemTime=1732779873873 2024-11-28T07:44:33,898 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:33,898 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:33,899 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1a4894ccf73e6e061b201e0caa5d0423, regionState=OPEN, openSeqNum=2, regionLocation=f64cf8bdf19b,36377,1732779871317 2024-11-28T07:44:33,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a4894ccf73e6e061b201e0caa5d0423, server=f64cf8bdf19b,36377,1732779871317 because future has completed 2024-11-28T07:44:33,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-28T07:44:33,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a4894ccf73e6e061b201e0caa5d0423, server=f64cf8bdf19b,36377,1732779871317 in 183 msec 2024-11-28T07:44:33,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-28T07:44:33,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732779873484, region=1a4894ccf73e6e061b201e0caa5d0423, ASSIGN in 345 msec 2024-11-28T07:44:33,908 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732779873484 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T07:44:33,909 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732779873484","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779873908"}]},"ts":"1732779873908"} 2024-11-28T07:44:33,911 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732779873484, state=ENABLED in hbase:meta 2024-11-28T07:44:33,912 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732779873484 execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T07:44:33,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732779873484 in 402 msec 2024-11-28T07:44:34,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:44:34,154 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732779873484 completed 2024-11-28T07:44:34,155 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732779873484 get assigned. Timeout = 60000ms 2024-11-28T07:44:34,158 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T07:44:34,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732779873484 assigned to meta. Checking AM states. 2024-11-28T07:44:34,165 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T07:44:34,165 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732779873484 assigned. 2024-11-28T07:44:34,173 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732779873484', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423., hostname=f64cf8bdf19b,36377,1732779871317, seqNum=2] 2024-11-28T07:44:34,183 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1a4894ccf73e6e061b201e0caa5d0423 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-28T07:44:34,208 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9374a6044c47486c8899c8671d01f68b 2024-11-28T07:44:34,250 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/.tmp/family/9374a6044c47486c8899c8671d01f68b is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-28T07:44:34,255 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/.tmp/family/9374a6044c47486c8899c8671d01f68b 2024-11-28T07:44:34,292 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/.tmp/family/9374a6044c47486c8899c8671d01f68b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/family/9374a6044c47486c8899c8671d01f68b 2024-11-28T07:44:34,294 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/family/9374a6044c47486c8899c8671d01f68b, delay=975 ms 2024-11-28T07:44:34,297 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/family/9374a6044c47486c8899c8671d01f68b, entries=2, sequenceid=6, filesize=4.9 K 2024-11-28T07:44:34,305 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 1a4894ccf73e6e061b201e0caa5d0423 in 119ms, sequenceid=6, compaction requested=false 2024-11-28T07:44:34,306 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732779873484' 2024-11-28T07:44:34,309 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1a4894ccf73e6e061b201e0caa5d0423: 2024-11-28T07:44:35,271 DEBUG [hfile-prefetch-1732779874294 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9374a6044c47486c8899c8671d01f68b_0. 2024-11-28T07:44:35,276 DEBUG [hfile-prefetch-1732779874294 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9374a6044c47486c8899c8671d01f68b_117. 2024-11-28T07:44:35,277 DEBUG [hfile-prefetch-1732779874294 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/family/9374a6044c47486c8899c8671d01f68b, with total blocks 2, and data blocks 1 2024-11-28T07:44:35,277 DEBUG [hfile-prefetch-1732779874294 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-28T07:44:35,278 DEBUG [hfile-prefetch-1732779874294 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/family/9374a6044c47486c8899c8671d01f68b, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-28T07:44:35,278 DEBUG [hfile-prefetch-1732779874294 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/family/9374a6044c47486c8899c8671d01f68b. Will sleep 100ms and try the verification again. 2024-11-28T07:44:35,284 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-28T07:44:35,284 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-28T07:44:35,285 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T07:44:35,285 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T07:44:35,301 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-28T07:44:35,301 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732779875284 2024-11-28T07:44:35,301 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-28T07:44:35,319 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=1a4894ccf73e6e061b201e0caa5d0423, source=f64cf8bdf19b,36377,1732779871317, destination=f64cf8bdf19b,34747,1732779871459, warming up region on f64cf8bdf19b,34747,1732779871459 2024-11-28T07:44:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T07:44:35,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=1a4894ccf73e6e061b201e0caa5d0423, source=f64cf8bdf19b,36377,1732779871317, destination=f64cf8bdf19b,34747,1732779871459, running balancer 2024-11-28T07:44:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732779873484, region=1a4894ccf73e6e061b201e0caa5d0423, REOPEN/MOVE 2024-11-28T07:44:35,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60641, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T07:44:35,322 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732779873484, region=1a4894ccf73e6e061b201e0caa5d0423, REOPEN/MOVE 2024-11-28T07:44:35,324 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1a4894ccf73e6e061b201e0caa5d0423, regionState=CLOSING, regionLocation=f64cf8bdf19b,36377,1732779871317 2024-11-28T07:44:35,324 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34747 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34747 {}] regionserver.HRegion(7855): Warmup {ENCODED => 1a4894ccf73e6e061b201e0caa5d0423, NAME => 'testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:44:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34747 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:44:35,325 INFO [StoreOpener-1a4894ccf73e6e061b201e0caa5d0423-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:35,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732779873484, region=1a4894ccf73e6e061b201e0caa5d0423, REOPEN/MOVE because future has completed 2024-11-28T07:44:35,326 INFO [StoreOpener-1a4894ccf73e6e061b201e0caa5d0423-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a4894ccf73e6e061b201e0caa5d0423 columnFamilyName family 2024-11-28T07:44:35,326 DEBUG [StoreOpener-1a4894ccf73e6e061b201e0caa5d0423-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:35,327 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-28T07:44:35,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a4894ccf73e6e061b201e0caa5d0423, server=f64cf8bdf19b,36377,1732779871317}] 2024-11-28T07:44:35,331 DEBUG [StoreOpener-1a4894ccf73e6e061b201e0caa5d0423-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/family/9374a6044c47486c8899c8671d01f68b 2024-11-28T07:44:35,331 INFO [StoreOpener-1a4894ccf73e6e061b201e0caa5d0423-1 {}] regionserver.HStore(327): Store=1a4894ccf73e6e061b201e0caa5d0423/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:44:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34747 {}] regionserver.HRegion(1722): Closing 1a4894ccf73e6e061b201e0caa5d0423, disabling compactions & flushes 2024-11-28T07:44:35,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34747 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34747 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34747 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. after waiting 0 ms 2024-11-28T07:44:35,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34747 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,332 DEBUG [StoreFileCloser-1a4894ccf73e6e061b201e0caa5d0423-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/family/9374a6044c47486c8899c8671d01f68b 2024-11-28T07:44:35,333 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34747 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34747 {}] regionserver.HRegion(1676): Region close journal for 1a4894ccf73e6e061b201e0caa5d0423: Waiting for close lock at 1732779875331Disabling compacts and flushes for region at 1732779875331Disabling writes for close at 1732779875331Writing region close event to WAL at 1732779875333 (+2 ms)Closed at 1732779875333 2024-11-28T07:44:35,334 WARN [hfile-prefetch-1732779874294 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/family/9374a6044c47486c8899c8671d01f68b, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-28T07:44:35,335 DEBUG [hfile-prefetch-1732779874294 {}] hfile.PrefetchExecutor(121): Prefetch completed for 9374a6044c47486c8899c8671d01f68b 2024-11-28T07:44:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43355 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-28T07:44:35,485 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:35,485 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-28T07:44:35,485 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 1a4894ccf73e6e061b201e0caa5d0423, disabling compactions & flushes 2024-11-28T07:44:35,486 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,486 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,486 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. after waiting 0 ms 2024-11-28T07:44:35,486 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,488 DEBUG [StoreFileCloser-1a4894ccf73e6e061b201e0caa5d0423-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 9374a6044c47486c8899c8671d01f68b, starting offset: 0, end offset: 9223372036854775807 2024-11-28T07:44:35,490 DEBUG [StoreFileCloser-1a4894ccf73e6e061b201e0caa5d0423-family-1 {}] bucket.BucketCache(856): removed key 9374a6044c47486c8899c8671d01f68b_0 from back map with offset lock 2094080 in the evict process 2024-11-28T07:44:35,495 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-28T07:44:35,497 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,497 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 1a4894ccf73e6e061b201e0caa5d0423: Waiting for close lock at 1732779875485Running coprocessor pre-close hooks at 1732779875485Disabling compacts and flushes for region at 1732779875485Disabling writes for close at 1732779875486 (+1 ms)Writing region close event to WAL at 1732779875490 (+4 ms)Running coprocessor post-close hooks at 1732779875496 (+6 ms)Closed at 1732779875497 (+1 ms) 2024-11-28T07:44:35,497 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 1a4894ccf73e6e061b201e0caa5d0423 move to f64cf8bdf19b,34747,1732779871459 record at close sequenceid=6 2024-11-28T07:44:35,500 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:35,500 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1a4894ccf73e6e061b201e0caa5d0423, regionState=CLOSED 2024-11-28T07:44:35,501 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-28T07:44:35,501 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-28T07:44:35,501 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T07:44:35,502 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T07:44:35,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a4894ccf73e6e061b201e0caa5d0423, server=f64cf8bdf19b,36377,1732779871317 because future has completed 2024-11-28T07:44:35,505 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-28T07:44:35,505 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732779875501 2024-11-28T07:44:35,505 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-28T07:44:35,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-28T07:44:35,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 1a4894ccf73e6e061b201e0caa5d0423, server=f64cf8bdf19b,36377,1732779871317 in 177 msec 2024-11-28T07:44:35,507 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732779873484, region=1a4894ccf73e6e061b201e0caa5d0423, REOPEN/MOVE; state=CLOSED, location=f64cf8bdf19b,34747,1732779871459; forceNewPlan=false, retain=false 2024-11-28T07:44:35,657 INFO [f64cf8bdf19b:43355 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-28T07:44:35,658 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1a4894ccf73e6e061b201e0caa5d0423, regionState=OPENING, regionLocation=f64cf8bdf19b,34747,1732779871459 2024-11-28T07:44:35,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732779873484, region=1a4894ccf73e6e061b201e0caa5d0423, REOPEN/MOVE because future has completed 2024-11-28T07:44:35,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a4894ccf73e6e061b201e0caa5d0423, server=f64cf8bdf19b,34747,1732779871459}] 2024-11-28T07:44:35,814 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,815 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a4894ccf73e6e061b201e0caa5d0423, NAME => 'testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:44:35,815 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732779873484 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:35,815 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:44:35,815 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:35,815 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:35,816 INFO [StoreOpener-1a4894ccf73e6e061b201e0caa5d0423-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:35,818 INFO [StoreOpener-1a4894ccf73e6e061b201e0caa5d0423-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a4894ccf73e6e061b201e0caa5d0423 columnFamilyName family 2024-11-28T07:44:35,818 DEBUG [StoreOpener-1a4894ccf73e6e061b201e0caa5d0423-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:35,820 DEBUG [StoreFileOpener-1a4894ccf73e6e061b201e0caa5d0423-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/family/9374a6044c47486c8899c8671d01f68b, delay=970 ms 2024-11-28T07:44:35,821 DEBUG [StoreOpener-1a4894ccf73e6e061b201e0caa5d0423-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/family/9374a6044c47486c8899c8671d01f68b 2024-11-28T07:44:35,821 INFO [StoreOpener-1a4894ccf73e6e061b201e0caa5d0423-1 {}] regionserver.HStore(327): Store=1a4894ccf73e6e061b201e0caa5d0423/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:44:35,821 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:35,822 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:35,823 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:35,823 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:35,823 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:35,827 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:35,828 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 1a4894ccf73e6e061b201e0caa5d0423; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61845652, jitterRate=-0.07842797040939331}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:44:35,828 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:35,829 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 1a4894ccf73e6e061b201e0caa5d0423: Running coprocessor pre-open hook at 1732779875816Writing region info on filesystem at 1732779875816Initializing all the Stores at 1732779875816Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779875816Cleaning up temporary data from old regions at 1732779875823 (+7 ms)Running coprocessor post-open hooks at 1732779875828 (+5 ms)Region opened successfully at 1732779875829 (+1 ms) 2024-11-28T07:44:35,829 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423., pid=9, masterSystemTime=1732779875812 2024-11-28T07:44:35,832 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,832 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,832 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1a4894ccf73e6e061b201e0caa5d0423, regionState=OPEN, openSeqNum=10, regionLocation=f64cf8bdf19b,34747,1732779871459 2024-11-28T07:44:35,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a4894ccf73e6e061b201e0caa5d0423, server=f64cf8bdf19b,34747,1732779871459 because future has completed 2024-11-28T07:44:35,838 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-28T07:44:35,838 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a4894ccf73e6e061b201e0caa5d0423, server=f64cf8bdf19b,34747,1732779871459 in 175 msec 2024-11-28T07:44:35,839 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732779873484, region=1a4894ccf73e6e061b201e0caa5d0423, REOPEN/MOVE in 517 msec 2024-11-28T07:44:35,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-28T07:44:35,875 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T07:44:35,875 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:44:35,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:35,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:35,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:44:35,880 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-28T07:44:35,880 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-28T07:44:35,881 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f64cf8bdf19b,43355,1732779870557 2024-11-28T07:44:35,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T07:44:35,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T07:44:35,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T07:44:35,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:35,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:35,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:35,883 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:44:35,884 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T07:44:35,884 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:44:35,884 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:44:35,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:35,884 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:44:35,884 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:44:35,885 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f64cf8bdf19b,36377,1732779871317' ***** 2024-11-28T07:44:35,885 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T07:44:35,885 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f64cf8bdf19b,34747,1732779871459' ***** 2024-11-28T07:44:35,885 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T07:44:35,885 INFO [RS:1;f64cf8bdf19b:34747 {}] bucket.BucketCache(1738): Disabling cache 2024-11-28T07:44:35,885 INFO [RS:0;f64cf8bdf19b:36377 {}] bucket.BucketCache(1738): Disabling cache 2024-11-28T07:44:35,885 INFO [RS:0;f64cf8bdf19b:36377 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/bucket.persistence 2024-11-28T07:44:35,885 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-28T07:44:35,886 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-28T07:44:35,886 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-28T07:44:35,886 INFO [RS:1;f64cf8bdf19b:34747 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/bucket.persistence 2024-11-28T07:44:35,886 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-28T07:44:35,886 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-28T07:44:35,886 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-28T07:44:35,886 INFO [RS:1;f64cf8bdf19b:34747 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-28T07:44:35,886 INFO [RS:0;f64cf8bdf19b:36377 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-28T07:44:35,886 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-28T07:44:35,886 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-28T07:44:35,886 DEBUG [RS:0;f64cf8bdf19b:36377 {}] bucket.BucketCache(1487): Thread RS:0;f64cf8bdf19b:36377 started persisting bucket cache to file 2024-11-28T07:44:35,887 DEBUG [RS:0;f64cf8bdf19b:36377 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T07:44:35,887 DEBUG [RS:0;f64cf8bdf19b:36377 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T07:44:35,890 DEBUG [RS:0;f64cf8bdf19b:36377 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-28T07:44:35,890 DEBUG [RS:0;f64cf8bdf19b:36377 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732779875886 2024-11-28T07:44:35,890 DEBUG [RS:0;f64cf8bdf19b:36377 {}] bucket.BucketCache(1509): Thread RS:0;f64cf8bdf19b:36377 finished persisting bucket cache to file, renaming 2024-11-28T07:44:35,890 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T07:44:35,891 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T07:44:35,891 INFO [RS:0;f64cf8bdf19b:36377 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T07:44:35,891 INFO [RS:0;f64cf8bdf19b:36377 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T07:44:35,891 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(959): stopping server f64cf8bdf19b,36377,1732779871317 2024-11-28T07:44:35,891 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T07:44:35,891 INFO [RS:0;f64cf8bdf19b:36377 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f64cf8bdf19b:36377. 2024-11-28T07:44:35,891 DEBUG [RS:0;f64cf8bdf19b:36377 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:44:35,891 DEBUG [RS:0;f64cf8bdf19b:36377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:35,891 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T07:44:35,891 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T07:44:35,891 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T07:44:35,892 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T07:44:35,892 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-28T07:44:35,892 DEBUG [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-28T07:44:35,892 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T07:44:35,892 DEBUG [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T07:44:35,893 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T07:44:35,893 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T07:44:35,893 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T07:44:35,893 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T07:44:35,893 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T07:44:35,893 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-28T07:44:35,896 DEBUG [RS:1;f64cf8bdf19b:34747 {}] bucket.BucketCache(1487): Thread RS:1;f64cf8bdf19b:34747 started persisting bucket cache to file 2024-11-28T07:44:35,896 DEBUG [RS:1;f64cf8bdf19b:34747 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T07:44:35,896 DEBUG [RS:1;f64cf8bdf19b:34747 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T07:44:35,900 DEBUG [RS:1;f64cf8bdf19b:34747 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-28T07:44:35,900 DEBUG [RS:1;f64cf8bdf19b:34747 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732779875896 2024-11-28T07:44:35,900 DEBUG [RS:1;f64cf8bdf19b:34747 {}] bucket.BucketCache(1509): Thread RS:1;f64cf8bdf19b:34747 finished persisting bucket cache to file, renaming 2024-11-28T07:44:35,901 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T07:44:35,901 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T07:44:35,901 INFO [RS:1;f64cf8bdf19b:34747 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T07:44:35,901 INFO [RS:1;f64cf8bdf19b:34747 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T07:44:35,901 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(3091): Received CLOSE for 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:35,901 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(959): stopping server f64cf8bdf19b,34747,1732779871459 2024-11-28T07:44:35,901 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T07:44:35,901 INFO [RS:1;f64cf8bdf19b:34747 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f64cf8bdf19b:34747. 2024-11-28T07:44:35,901 DEBUG [RS:1;f64cf8bdf19b:34747 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:44:35,902 DEBUG [RS:1;f64cf8bdf19b:34747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:35,902 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T07:44:35,902 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-28T07:44:35,902 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1a4894ccf73e6e061b201e0caa5d0423, disabling compactions & flushes 2024-11-28T07:44:35,902 DEBUG [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(1325): Online Regions={1a4894ccf73e6e061b201e0caa5d0423=testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423.} 2024-11-28T07:44:35,902 DEBUG [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(1351): Waiting on 1a4894ccf73e6e061b201e0caa5d0423 2024-11-28T07:44:35,902 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,902 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,902 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. after waiting 0 ms 2024-11-28T07:44:35,902 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,904 DEBUG [StoreFileCloser-1a4894ccf73e6e061b201e0caa5d0423-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/family/9374a6044c47486c8899c8671d01f68b 2024-11-28T07:44:35,904 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 802efa10c7c348bd9721f5cd778c64d0 2024-11-28T07:44:35,909 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnRegionMove1732779873484/1a4894ccf73e6e061b201e0caa5d0423/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-28T07:44:35,910 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,910 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1a4894ccf73e6e061b201e0caa5d0423: Waiting for close lock at 1732779875901Running coprocessor pre-close hooks at 1732779875902 (+1 ms)Disabling compacts and flushes for region at 1732779875902Disabling writes for close at 1732779875902Writing region close event to WAL at 1732779875905 (+3 ms)Running coprocessor post-close hooks at 1732779875910 (+5 ms)Closed at 1732779875910 2024-11-28T07:44:35,910 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423. 2024-11-28T07:44:35,922 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/info/802efa10c7c348bd9721f5cd778c64d0 is 205, key is testBlockEvictionOnRegionMove1732779873484,,1732779873504.1a4894ccf73e6e061b201e0caa5d0423./info:regioninfo/1732779875832/Put/seqid=0 2024-11-28T07:44:35,923 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/info/802efa10c7c348bd9721f5cd778c64d0 2024-11-28T07:44:35,933 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1c19f4f1ba6a43e29d79ef6d8a4690b5 2024-11-28T07:44:35,946 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/ns/1c19f4f1ba6a43e29d79ef6d8a4690b5 is 43, key is default/ns:d/1732779873302/Put/seqid=0 2024-11-28T07:44:35,947 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/ns/1c19f4f1ba6a43e29d79ef6d8a4690b5 2024-11-28T07:44:35,956 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b2580d778227406481bee9dfadab75fe 2024-11-28T07:44:35,970 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/table/b2580d778227406481bee9dfadab75fe is 78, key is testBlockEvictionOnRegionMove1732779873484/table:state/1732779873908/Put/seqid=0 2024-11-28T07:44:35,971 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/table/b2580d778227406481bee9dfadab75fe 2024-11-28T07:44:35,974 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/info/802efa10c7c348bd9721f5cd778c64d0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/info/802efa10c7c348bd9721f5cd778c64d0 2024-11-28T07:44:35,976 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/info/802efa10c7c348bd9721f5cd778c64d0, delay=976 ms 2024-11-28T07:44:35,977 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/info/802efa10c7c348bd9721f5cd778c64d0, entries=15, sequenceid=15, filesize=7.6 K 2024-11-28T07:44:35,977 INFO [regionserver/f64cf8bdf19b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:44:35,977 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/ns/1c19f4f1ba6a43e29d79ef6d8a4690b5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/ns/1c19f4f1ba6a43e29d79ef6d8a4690b5 2024-11-28T07:44:35,978 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/ns/1c19f4f1ba6a43e29d79ef6d8a4690b5, delay=980 ms 2024-11-28T07:44:35,978 INFO [regionserver/f64cf8bdf19b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:44:35,980 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/ns/1c19f4f1ba6a43e29d79ef6d8a4690b5, entries=2, sequenceid=15, filesize=5.0 K 2024-11-28T07:44:35,981 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/table/b2580d778227406481bee9dfadab75fe as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/table/b2580d778227406481bee9dfadab75fe 2024-11-28T07:44:35,982 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/table/b2580d778227406481bee9dfadab75fe, delay=947 ms 2024-11-28T07:44:35,983 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/table/b2580d778227406481bee9dfadab75fe, entries=2, sequenceid=15, filesize=5.3 K 2024-11-28T07:44:35,984 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=15, compaction requested=false 2024-11-28T07:44:35,984 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-28T07:44:35,985 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/info/802efa10c7c348bd9721f5cd778c64d0 2024-11-28T07:44:35,986 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/ns/1c19f4f1ba6a43e29d79ef6d8a4690b5 2024-11-28T07:44:35,987 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/table/b2580d778227406481bee9dfadab75fe 2024-11-28T07:44:35,992 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-28T07:44:35,993 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T07:44:35,993 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T07:44:35,993 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732779875892Running coprocessor pre-close hooks at 1732779875893 (+1 ms)Disabling compacts and flushes for region at 1732779875893Disabling writes for close at 1732779875893Obtaining lock to block concurrent updates at 1732779875893Preparing flush snapshotting stores in 1588230740 at 1732779875893Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732779875894 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732779875894Flushing 1588230740/info: creating writer at 1732779875894Flushing 1588230740/info: appending metadata at 1732779875921 (+27 ms)Flushing 1588230740/info: closing flushed file at 1732779875921Flushing 1588230740/ns: creating writer at 1732779875926 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732779875945 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732779875946 (+1 ms)Flushing 1588230740/table: creating writer at 1732779875950 (+4 ms)Flushing 1588230740/table: appending metadata at 1732779875969 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732779875970 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71b429bd: reopening flushed file at 1732779875974 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fa4ae7: reopening flushed file at 1732779875977 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7efb6455: reopening flushed file at 1732779875980 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=15, compaction requested=false at 1732779875984 (+4 ms)Writing region close event to WAL at 1732779875987 (+3 ms)Running coprocessor post-close hooks at 1732779875992 (+5 ms)Closed at 1732779875993 (+1 ms) 2024-11-28T07:44:35,994 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T07:44:36,093 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(976): stopping server f64cf8bdf19b,36377,1732779871317; all regions closed. 2024-11-28T07:44:36,097 DEBUG [RS:0;f64cf8bdf19b:36377 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/oldWALs 2024-11-28T07:44:36,098 INFO [RS:0;f64cf8bdf19b:36377 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f64cf8bdf19b%2C36377%2C1732779871317.meta:.meta(num 1732779873167) 2024-11-28T07:44:36,099 DEBUG [RS:0;f64cf8bdf19b:36377 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/oldWALs 2024-11-28T07:44:36,099 INFO [RS:0;f64cf8bdf19b:36377 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f64cf8bdf19b%2C36377%2C1732779871317:(num 1732779872846) 2024-11-28T07:44:36,100 DEBUG [RS:0;f64cf8bdf19b:36377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:36,100 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:44:36,100 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T07:44:36,100 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.ChoreService(370): Chore service for: regionserver/f64cf8bdf19b:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-28T07:44:36,100 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T07:44:36,100 INFO [regionserver/f64cf8bdf19b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T07:44:36,101 INFO [RS:0;f64cf8bdf19b:36377 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36377 2024-11-28T07:44:36,102 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(976): stopping server f64cf8bdf19b,34747,1732779871459; all regions closed. 2024-11-28T07:44:36,104 DEBUG [RS:1;f64cf8bdf19b:34747 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/oldWALs 2024-11-28T07:44:36,104 INFO [RS:1;f64cf8bdf19b:34747 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f64cf8bdf19b%2C34747%2C1732779871459:(num 1732779872846) 2024-11-28T07:44:36,104 DEBUG [RS:1;f64cf8bdf19b:34747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:36,104 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:44:36,104 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T07:44:36,104 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.ChoreService(370): Chore service for: regionserver/f64cf8bdf19b:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-28T07:44:36,105 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T07:44:36,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T07:44:36,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f64cf8bdf19b,36377,1732779871317 2024-11-28T07:44:36,105 INFO [regionserver/f64cf8bdf19b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T07:44:36,105 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T07:44:36,105 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T07:44:36,105 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T07:44:36,105 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T07:44:36,105 INFO [RS:1;f64cf8bdf19b:34747 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34747 2024-11-28T07:44:36,106 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f64cf8bdf19b,36377,1732779871317] 2024-11-28T07:44:36,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f64cf8bdf19b,34747,1732779871459 2024-11-28T07:44:36,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T07:44:36,109 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T07:44:36,109 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f64cf8bdf19b,36377,1732779871317 already deleted, retry=false 2024-11-28T07:44:36,110 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f64cf8bdf19b,36377,1732779871317 expired; onlineServers=1 2024-11-28T07:44:36,110 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f64cf8bdf19b,34747,1732779871459] 2024-11-28T07:44:36,112 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f64cf8bdf19b,34747,1732779871459 already deleted, retry=false 2024-11-28T07:44:36,112 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f64cf8bdf19b,34747,1732779871459 expired; onlineServers=0 2024-11-28T07:44:36,112 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f64cf8bdf19b,43355,1732779870557' ***** 2024-11-28T07:44:36,112 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-28T07:44:36,112 INFO [M:0;f64cf8bdf19b:43355 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T07:44:36,112 INFO [M:0;f64cf8bdf19b:43355 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T07:44:36,112 DEBUG [M:0;f64cf8bdf19b:43355 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-28T07:44:36,112 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-28T07:44:36,112 DEBUG [M:0;f64cf8bdf19b:43355 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-28T07:44:36,112 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.small.0-1732779872459 {}] cleaner.HFileCleaner(306): Exit Thread[master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.small.0-1732779872459,5,FailOnTimeoutGroup] 2024-11-28T07:44:36,112 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.large.0-1732779872459 {}] cleaner.HFileCleaner(306): Exit Thread[master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.large.0-1732779872459,5,FailOnTimeoutGroup] 2024-11-28T07:44:36,113 INFO [M:0;f64cf8bdf19b:43355 {}] hbase.ChoreService(370): Chore service for: master/f64cf8bdf19b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-28T07:44:36,113 INFO [M:0;f64cf8bdf19b:43355 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T07:44:36,113 DEBUG [M:0;f64cf8bdf19b:43355 {}] master.HMaster(1795): Stopping service threads 2024-11-28T07:44:36,113 INFO [M:0;f64cf8bdf19b:43355 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-28T07:44:36,113 INFO [M:0;f64cf8bdf19b:43355 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:44:36,114 INFO [M:0;f64cf8bdf19b:43355 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:44:36,114 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-28T07:44:36,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-28T07:44:36,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:36,116 DEBUG [M:0;f64cf8bdf19b:43355 {}] zookeeper.ZKUtil(347): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-28T07:44:36,117 WARN [M:0;f64cf8bdf19b:43355 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-28T07:44:36,117 INFO [M:0;f64cf8bdf19b:43355 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/.lastflushedseqids 2024-11-28T07:44:36,129 INFO [M:0;f64cf8bdf19b:43355 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:44:36,129 INFO [M:0;f64cf8bdf19b:43355 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-28T07:44:36,129 DEBUG [M:0;f64cf8bdf19b:43355 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T07:44:36,129 INFO [M:0;f64cf8bdf19b:43355 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:44:36,129 DEBUG [M:0;f64cf8bdf19b:43355 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:44:36,129 DEBUG [M:0;f64cf8bdf19b:43355 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T07:44:36,129 DEBUG [M:0;f64cf8bdf19b:43355 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:44:36,130 INFO [M:0;f64cf8bdf19b:43355 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-28T07:44:36,141 DEBUG [M:0;f64cf8bdf19b:43355 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 572ca271f4e44b21ac7bc9da3f460879 2024-11-28T07:44:36,155 DEBUG [M:0;f64cf8bdf19b:43355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/572ca271f4e44b21ac7bc9da3f460879 is 82, key is hbase:meta,,1/info:regioninfo/1732779873234/Put/seqid=0 2024-11-28T07:44:36,156 INFO [M:0;f64cf8bdf19b:43355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/572ca271f4e44b21ac7bc9da3f460879 2024-11-28T07:44:36,165 DEBUG [M:0;f64cf8bdf19b:43355 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d3cb6c4b5de34dcfa23bbc746192b52c 2024-11-28T07:44:36,179 DEBUG [M:0;f64cf8bdf19b:43355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d3cb6c4b5de34dcfa23bbc746192b52c is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732779873913/Put/seqid=0 2024-11-28T07:44:36,180 INFO [M:0;f64cf8bdf19b:43355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d3cb6c4b5de34dcfa23bbc746192b52c 2024-11-28T07:44:36,190 DEBUG [M:0;f64cf8bdf19b:43355 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 69b625ad0e7f414397ccef6a2aae2b67 2024-11-28T07:44:36,203 DEBUG [M:0;f64cf8bdf19b:43355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/69b625ad0e7f414397ccef6a2aae2b67 is 69, key is f64cf8bdf19b,34747,1732779871459/rs:state/1732779872589/Put/seqid=0 2024-11-28T07:44:36,203 INFO [M:0;f64cf8bdf19b:43355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/69b625ad0e7f414397ccef6a2aae2b67 2024-11-28T07:44:36,205 DEBUG [M:0;f64cf8bdf19b:43355 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/572ca271f4e44b21ac7bc9da3f460879 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/572ca271f4e44b21ac7bc9da3f460879 2024-11-28T07:44:36,206 INFO [M:0;f64cf8bdf19b:43355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/572ca271f4e44b21ac7bc9da3f460879, entries=8, sequenceid=84, filesize=5.5 K 2024-11-28T07:44:36,207 DEBUG [M:0;f64cf8bdf19b:43355 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d3cb6c4b5de34dcfa23bbc746192b52c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d3cb6c4b5de34dcfa23bbc746192b52c 2024-11-28T07:44:36,208 INFO [M:0;f64cf8bdf19b:43355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d3cb6c4b5de34dcfa23bbc746192b52c, entries=9, sequenceid=84, filesize=6.5 K 2024-11-28T07:44:36,209 DEBUG [M:0;f64cf8bdf19b:43355 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/69b625ad0e7f414397ccef6a2aae2b67 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/69b625ad0e7f414397ccef6a2aae2b67 2024-11-28T07:44:36,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:44:36,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36377-0x1015d8872c50001, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:44:36,210 INFO [RS:0;f64cf8bdf19b:36377 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T07:44:36,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:44:36,210 INFO [RS:1;f64cf8bdf19b:34747 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T07:44:36,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34747-0x1015d8872c50002, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:44:36,210 INFO [RS:0;f64cf8bdf19b:36377 {}] regionserver.HRegionServer(1031): Exiting; stopping=f64cf8bdf19b,36377,1732779871317; zookeeper connection closed. 2024-11-28T07:44:36,211 INFO [RS:1;f64cf8bdf19b:34747 {}] regionserver.HRegionServer(1031): Exiting; stopping=f64cf8bdf19b,34747,1732779871459; zookeeper connection closed. 2024-11-28T07:44:36,211 INFO [M:0;f64cf8bdf19b:43355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/69b625ad0e7f414397ccef6a2aae2b67, entries=2, sequenceid=84, filesize=5.1 K 2024-11-28T07:44:36,211 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6c8674ff {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6c8674ff 2024-11-28T07:44:36,211 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7d1071de {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7d1071de 2024-11-28T07:44:36,212 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-28T07:44:36,212 INFO [M:0;f64cf8bdf19b:43355 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 83ms, sequenceid=84, compaction requested=false 2024-11-28T07:44:36,217 INFO [M:0;f64cf8bdf19b:43355 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:44:36,217 DEBUG [M:0;f64cf8bdf19b:43355 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732779876129Disabling compacts and flushes for region at 1732779876129Disabling writes for close at 1732779876129Obtaining lock to block concurrent updates at 1732779876130 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732779876130Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732779876130Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732779876131 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732779876131Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732779876155 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732779876155Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732779876158 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732779876179 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732779876179Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732779876183 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732779876202 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732779876202Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37791a09: reopening flushed file at 1732779876205 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51161dc7: reopening flushed file at 1732779876207 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48d684ca: reopening flushed file at 1732779876209 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 83ms, sequenceid=84, compaction requested=false at 1732779876212 (+3 ms)Writing region close event to WAL at 1732779876217 (+5 ms)Closed at 1732779876217 2024-11-28T07:44:36,219 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T07:44:36,219 INFO [M:0;f64cf8bdf19b:43355 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-28T07:44:36,219 INFO [M:0;f64cf8bdf19b:43355 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43355 2024-11-28T07:44:36,219 INFO [M:0;f64cf8bdf19b:43355 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T07:44:36,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:44:36,321 INFO [M:0;f64cf8bdf19b:43355 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T07:44:36,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43355-0x1015d8872c50000, quorum=127.0.0.1:50908, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:44:36,326 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-28T07:44:36,345 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-28T07:44:36,346 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:36,354 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/f64cf8bdf19b:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f64cf8bdf19b:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732779875978 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732779875820 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f64cf8bdf19b:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732779875976 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732779874294 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/f64cf8bdf19b:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=339 (was 299) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7106 (was 7496) 2024-11-28T07:44:36,360 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=339, ProcessCount=11, AvailableMemoryMB=7105 2024-11-28T07:44:36,360 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/hadoop-log-dir so I do NOT create it in target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376 2024-11-28T07:44:36,360 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/hadoop-tmp-dir so I do NOT create it in target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376 2024-11-28T07:44:36,361 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:36,363 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/cluster_9bca592a-e17a-73e7-3c7b-34eed6670f23, deleteOnExit=true 2024-11-28T07:44:36,366 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/cluster_9bca592a-e17a-73e7-3c7b-34eed6670f23/zookeeper_0, clientPort=64536, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/cluster_9bca592a-e17a-73e7-3c7b-34eed6670f23/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/cluster_9bca592a-e17a-73e7-3c7b-34eed6670f23/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-28T07:44:36,367 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64536 2024-11-28T07:44:36,379 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597 with version=8 2024-11-28T07:44:36,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/hbase-staging 2024-11-28T07:44:36,382 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f64cf8bdf19b:0 server-side Connection retries=45 2024-11-28T07:44:36,382 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:36,382 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:36,382 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T07:44:36,382 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:36,382 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T07:44:36,382 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-28T07:44:36,382 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T07:44:36,383 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43047 2024-11-28T07:44:36,384 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43047 connecting to ZooKeeper ensemble=127.0.0.1:64536 2024-11-28T07:44:36,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:430470x0, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T07:44:36,391 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43047-0x1015d888a9d0000 connected 2024-11-28T07:44:36,414 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:36,414 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:36,415 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:44:36,415 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597, hbase.cluster.distributed=false 2024-11-28T07:44:36,417 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T07:44:36,417 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43047 2024-11-28T07:44:36,417 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43047 2024-11-28T07:44:36,418 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43047 2024-11-28T07:44:36,418 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43047 2024-11-28T07:44:36,418 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43047 2024-11-28T07:44:36,434 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f64cf8bdf19b:0 server-side Connection retries=45 2024-11-28T07:44:36,434 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:36,434 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:36,435 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T07:44:36,435 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:36,435 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T07:44:36,435 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T07:44:36,435 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T07:44:36,436 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42741 2024-11-28T07:44:36,437 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42741 connecting to ZooKeeper ensemble=127.0.0.1:64536 2024-11-28T07:44:36,437 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:36,437 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:36,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:427410x0, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T07:44:36,440 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42741-0x1015d888a9d0001 connected 2024-11-28T07:44:36,440 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:44:36,441 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T07:44:36,441 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/bucket.cache 2024-11-28T07:44:36,442 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-28T07:44:36,442 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-28T07:44:36,442 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-28T07:44:36,442 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-28T07:44:36,442 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-28T07:44:36,442 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-28T07:44:36,443 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T07:44:36,443 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-28T07:44:36,443 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/bucket.persistence is complete. 2024-11-28T07:44:36,443 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T07:44:36,444 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T07:44:36,445 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42741 2024-11-28T07:44:36,445 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42741 2024-11-28T07:44:36,445 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42741 2024-11-28T07:44:36,446 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42741 2024-11-28T07:44:36,446 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42741 2024-11-28T07:44:36,461 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f64cf8bdf19b:0 server-side Connection retries=45 2024-11-28T07:44:36,461 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:36,461 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:36,462 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T07:44:36,462 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:36,462 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T07:44:36,462 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T07:44:36,462 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T07:44:36,462 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42865 2024-11-28T07:44:36,464 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42865 connecting to ZooKeeper ensemble=127.0.0.1:64536 2024-11-28T07:44:36,464 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:36,464 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:36,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:428650x0, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T07:44:36,468 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:428650x0, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:44:36,468 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42865-0x1015d888a9d0002 connected 2024-11-28T07:44:36,468 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T07:44:36,471 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/bucket.cache 2024-11-28T07:44:36,471 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-28T07:44:36,471 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-28T07:44:36,471 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-28T07:44:36,471 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-28T07:44:36,472 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-28T07:44:36,472 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-28T07:44:36,472 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-28T07:44:36,472 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/bucket.persistence is complete. 2024-11-28T07:44:36,472 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T07:44:36,473 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T07:44:36,474 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T07:44:36,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42865 2024-11-28T07:44:36,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42865 2024-11-28T07:44:36,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42865 2024-11-28T07:44:36,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42865 2024-11-28T07:44:36,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42865 2024-11-28T07:44:36,499 DEBUG [M:0;f64cf8bdf19b:43047 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f64cf8bdf19b:43047 2024-11-28T07:44:36,499 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f64cf8bdf19b,43047,1732779876381 2024-11-28T07:44:36,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:44:36,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:44:36,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:44:36,506 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f64cf8bdf19b,43047,1732779876381 2024-11-28T07:44:36,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T07:44:36,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:36,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:36,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T07:44:36,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:36,508 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T07:44:36,509 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f64cf8bdf19b,43047,1732779876381 from backup master directory 2024-11-28T07:44:36,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f64cf8bdf19b,43047,1732779876381 2024-11-28T07:44:36,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:44:36,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:44:36,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:44:36,511 WARN [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T07:44:36,511 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f64cf8bdf19b,43047,1732779876381 2024-11-28T07:44:36,512 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/hbase.id] with ID: 47067c86-19cb-49df-9194-087e1d2051ef 2024-11-28T07:44:36,512 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/.tmp/hbase.id 2024-11-28T07:44:36,520 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/hbase.id] 2024-11-28T07:44:36,558 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:36,559 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-28T07:44:36,561 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-28T07:44:36,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:36,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:36,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:36,568 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T07:44:36,569 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-28T07:44:36,569 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:44:36,589 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store 2024-11-28T07:44:36,596 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:44:36,596 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T07:44:36,596 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:44:36,596 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:44:36,596 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T07:44:36,596 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:44:36,596 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:44:36,596 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732779876596Disabling compacts and flushes for region at 1732779876596Disabling writes for close at 1732779876596Writing region close event to WAL at 1732779876596Closed at 1732779876596 2024-11-28T07:44:36,596 WARN [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/.initializing 2024-11-28T07:44:36,596 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/WALs/f64cf8bdf19b,43047,1732779876381 2024-11-28T07:44:36,602 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T07:44:36,605 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f64cf8bdf19b%2C43047%2C1732779876381, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/WALs/f64cf8bdf19b,43047,1732779876381, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/oldWALs, maxLogs=10 2024-11-28T07:44:36,623 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/WALs/f64cf8bdf19b,43047,1732779876381/f64cf8bdf19b%2C43047%2C1732779876381.1732779876605 2024-11-28T07:44:36,623 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T07:44:36,623 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:44:36,623 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:44:36,623 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:36,623 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:36,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:36,629 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-28T07:44:36,629 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:36,629 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:36,629 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:36,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-28T07:44:36,633 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:36,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:44:36,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:36,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-28T07:44:36,636 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:36,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:44:36,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:36,640 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-28T07:44:36,640 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:36,640 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:44:36,640 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:36,640 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:36,641 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:36,641 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:36,641 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:36,641 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T07:44:36,641 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:44:36,647 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:44:36,648 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64148509, jitterRate=-0.04411272704601288}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T07:44:36,648 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732779876624Initializing all the Stores at 1732779876624Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779876624Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779876624Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779876624Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779876624Cleaning up temporary data from old regions at 1732779876641 (+17 ms)Region opened successfully at 1732779876648 (+7 ms) 2024-11-28T07:44:36,649 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-28T07:44:36,652 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f73fbfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f64cf8bdf19b/172.17.0.2:0 2024-11-28T07:44:36,654 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-28T07:44:36,654 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-28T07:44:36,654 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-28T07:44:36,654 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:44:36,654 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T07:44:36,655 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-28T07:44:36,655 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-28T07:44:36,656 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-28T07:44:36,657 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-28T07:44:36,660 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-28T07:44:36,660 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-28T07:44:36,661 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-28T07:44:36,662 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-28T07:44:36,662 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-28T07:44:36,663 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-28T07:44:36,664 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-28T07:44:36,665 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-28T07:44:36,666 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-28T07:44:36,669 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-28T07:44:36,671 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-28T07:44:36,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T07:44:36,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T07:44:36,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T07:44:36,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:36,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:36,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:36,674 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f64cf8bdf19b,43047,1732779876381, sessionid=0x1015d888a9d0000, setting cluster-up flag (Was=false) 2024-11-28T07:44:36,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:36,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:36,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:36,684 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-28T07:44:36,686 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f64cf8bdf19b,43047,1732779876381 2024-11-28T07:44:36,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:36,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:36,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:36,699 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-28T07:44:36,701 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f64cf8bdf19b,43047,1732779876381 2024-11-28T07:44:36,701 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-28T07:44:36,703 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:44:36,703 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-28T07:44:36,703 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-28T07:44:36,703 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f64cf8bdf19b,43047,1732779876381 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-28T07:44:36,705 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f64cf8bdf19b:0, corePoolSize=5, maxPoolSize=5 2024-11-28T07:44:36,705 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f64cf8bdf19b:0, corePoolSize=5, maxPoolSize=5 2024-11-28T07:44:36,705 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f64cf8bdf19b:0, corePoolSize=5, maxPoolSize=5 2024-11-28T07:44:36,705 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f64cf8bdf19b:0, corePoolSize=5, maxPoolSize=5 2024-11-28T07:44:36,705 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f64cf8bdf19b:0, corePoolSize=10, maxPoolSize=10 2024-11-28T07:44:36,705 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,705 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f64cf8bdf19b:0, corePoolSize=2, maxPoolSize=2 2024-11-28T07:44:36,705 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,707 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T07:44:36,707 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732779906707 2024-11-28T07:44:36,707 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-28T07:44:36,707 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-28T07:44:36,707 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-28T07:44:36,707 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-28T07:44:36,707 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-28T07:44:36,707 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-28T07:44:36,707 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-28T07:44:36,707 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:36,708 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,707 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-28T07:44:36,709 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-28T07:44:36,709 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-28T07:44:36,709 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-28T07:44:36,711 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-28T07:44:36,711 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-28T07:44:36,712 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.large.0-1732779876711,5,FailOnTimeoutGroup] 2024-11-28T07:44:36,715 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.small.0-1732779876712,5,FailOnTimeoutGroup] 2024-11-28T07:44:36,715 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,716 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-28T07:44:36,716 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,716 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,727 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-28T07:44:36,727 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597 2024-11-28T07:44:36,734 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:44:36,735 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-28T07:44:36,738 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T07:44:36,738 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:36,738 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:36,738 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-28T07:44:36,742 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T07:44:36,742 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:36,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:36,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-28T07:44:36,745 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T07:44:36,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:36,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:36,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-28T07:44:36,749 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T07:44:36,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:36,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:36,749 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T07:44:36,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740 2024-11-28T07:44:36,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740 2024-11-28T07:44:36,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T07:44:36,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T07:44:36,751 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T07:44:36,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T07:44:36,757 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:44:36,757 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69036274, jitterRate=0.02872064709663391}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T07:44:36,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732779876734Initializing all the Stores at 1732779876734Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779876734Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779876735 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779876735Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779876735Cleaning up temporary data from old regions at 1732779876750 (+15 ms)Region opened successfully at 1732779876758 (+8 ms) 2024-11-28T07:44:36,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T07:44:36,758 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T07:44:36,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T07:44:36,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T07:44:36,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T07:44:36,759 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T07:44:36,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732779876758Disabling compacts and flushes for region at 1732779876758Disabling writes for close at 1732779876758Writing region close event to WAL at 1732779876759 (+1 ms)Closed at 1732779876759 2024-11-28T07:44:36,760 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T07:44:36,760 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-28T07:44:36,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T07:44:36,761 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:44:36,762 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-28T07:44:36,783 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer(746): ClusterId : 47067c86-19cb-49df-9194-087e1d2051ef 2024-11-28T07:44:36,783 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.HRegionServer(746): ClusterId : 47067c86-19cb-49df-9194-087e1d2051ef 2024-11-28T07:44:36,783 DEBUG [RS:1;f64cf8bdf19b:42865 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T07:44:36,783 DEBUG [RS:0;f64cf8bdf19b:42741 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T07:44:36,786 DEBUG [RS:1;f64cf8bdf19b:42865 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T07:44:36,786 DEBUG [RS:0;f64cf8bdf19b:42741 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T07:44:36,786 DEBUG [RS:1;f64cf8bdf19b:42865 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T07:44:36,786 DEBUG [RS:0;f64cf8bdf19b:42741 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T07:44:36,789 DEBUG [RS:1;f64cf8bdf19b:42865 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T07:44:36,789 DEBUG [RS:0;f64cf8bdf19b:42741 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T07:44:36,789 DEBUG [RS:0;f64cf8bdf19b:42741 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38d31d7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f64cf8bdf19b/172.17.0.2:0 2024-11-28T07:44:36,789 DEBUG [RS:1;f64cf8bdf19b:42865 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40141449, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f64cf8bdf19b/172.17.0.2:0 2024-11-28T07:44:36,802 DEBUG [RS:0;f64cf8bdf19b:42741 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f64cf8bdf19b:42741 2024-11-28T07:44:36,802 DEBUG [RS:1;f64cf8bdf19b:42865 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f64cf8bdf19b:42865 2024-11-28T07:44:36,803 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T07:44:36,803 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T07:44:36,803 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T07:44:36,803 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T07:44:36,803 DEBUG [RS:1;f64cf8bdf19b:42865 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T07:44:36,803 DEBUG [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T07:44:36,804 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer(2659): reportForDuty to master=f64cf8bdf19b,43047,1732779876381 with port=42741, startcode=1732779876434 2024-11-28T07:44:36,804 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.HRegionServer(2659): reportForDuty to master=f64cf8bdf19b,43047,1732779876381 with port=42865, startcode=1732779876461 2024-11-28T07:44:36,804 DEBUG [RS:0;f64cf8bdf19b:42741 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T07:44:36,804 DEBUG [RS:1;f64cf8bdf19b:42865 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T07:44:36,807 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60211, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T07:44:36,808 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40355, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T07:44:36,808 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43047 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f64cf8bdf19b,42865,1732779876461 2024-11-28T07:44:36,808 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43047 {}] master.ServerManager(517): Registering regionserver=f64cf8bdf19b,42865,1732779876461 2024-11-28T07:44:36,810 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43047 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f64cf8bdf19b,42741,1732779876434 2024-11-28T07:44:36,810 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43047 {}] master.ServerManager(517): Registering regionserver=f64cf8bdf19b,42741,1732779876434 2024-11-28T07:44:36,810 DEBUG [RS:1;f64cf8bdf19b:42865 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597 2024-11-28T07:44:36,810 DEBUG [RS:1;f64cf8bdf19b:42865 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-28T07:44:36,810 DEBUG [RS:1;f64cf8bdf19b:42865 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T07:44:36,811 DEBUG [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597 2024-11-28T07:44:36,811 DEBUG [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-28T07:44:36,812 DEBUG [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T07:44:36,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T07:44:36,820 DEBUG [RS:1;f64cf8bdf19b:42865 {}] zookeeper.ZKUtil(111): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f64cf8bdf19b,42865,1732779876461 2024-11-28T07:44:36,820 WARN [RS:1;f64cf8bdf19b:42865 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T07:44:36,820 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f64cf8bdf19b,42865,1732779876461] 2024-11-28T07:44:36,820 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f64cf8bdf19b,42741,1732779876434] 2024-11-28T07:44:36,820 INFO [RS:1;f64cf8bdf19b:42865 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:44:36,820 DEBUG [RS:0;f64cf8bdf19b:42741 {}] zookeeper.ZKUtil(111): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f64cf8bdf19b,42741,1732779876434 2024-11-28T07:44:36,820 DEBUG [RS:1;f64cf8bdf19b:42865 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,42865,1732779876461 2024-11-28T07:44:36,820 WARN [RS:0;f64cf8bdf19b:42741 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T07:44:36,821 INFO [RS:0;f64cf8bdf19b:42741 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:44:36,821 DEBUG [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,42741,1732779876434 2024-11-28T07:44:36,828 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T07:44:36,830 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T07:44:36,830 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T07:44:36,832 INFO [RS:1;f64cf8bdf19b:42865 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T07:44:36,832 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T07:44:36,832 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,833 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T07:44:36,833 INFO [RS:0;f64cf8bdf19b:42741 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T07:44:36,833 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,833 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T07:44:36,833 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T07:44:36,833 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T07:44:36,834 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,834 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,834 DEBUG [RS:0;f64cf8bdf19b:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,834 DEBUG [RS:1;f64cf8bdf19b:42865 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,834 DEBUG [RS:0;f64cf8bdf19b:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,834 DEBUG [RS:0;f64cf8bdf19b:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,834 DEBUG [RS:1;f64cf8bdf19b:42865 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,834 DEBUG [RS:0;f64cf8bdf19b:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,834 DEBUG [RS:0;f64cf8bdf19b:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,834 DEBUG [RS:1;f64cf8bdf19b:42865 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,834 DEBUG [RS:0;f64cf8bdf19b:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f64cf8bdf19b:0, corePoolSize=2, maxPoolSize=2 2024-11-28T07:44:36,834 DEBUG [RS:1;f64cf8bdf19b:42865 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,834 DEBUG [RS:0;f64cf8bdf19b:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,834 DEBUG [RS:0;f64cf8bdf19b:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,834 DEBUG [RS:1;f64cf8bdf19b:42865 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,834 DEBUG [RS:0;f64cf8bdf19b:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,835 DEBUG [RS:1;f64cf8bdf19b:42865 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f64cf8bdf19b:0, corePoolSize=2, maxPoolSize=2 2024-11-28T07:44:36,835 DEBUG [RS:0;f64cf8bdf19b:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,835 DEBUG [RS:1;f64cf8bdf19b:42865 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,835 DEBUG [RS:0;f64cf8bdf19b:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,835 DEBUG [RS:1;f64cf8bdf19b:42865 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,835 DEBUG [RS:0;f64cf8bdf19b:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,835 DEBUG [RS:0;f64cf8bdf19b:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f64cf8bdf19b:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:44:36,835 DEBUG [RS:0;f64cf8bdf19b:42741 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:44:36,835 DEBUG [RS:1;f64cf8bdf19b:42865 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,835 DEBUG [RS:1;f64cf8bdf19b:42865 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,835 DEBUG [RS:1;f64cf8bdf19b:42865 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,835 DEBUG [RS:1;f64cf8bdf19b:42865 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:36,835 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,835 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,835 DEBUG [RS:1;f64cf8bdf19b:42865 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f64cf8bdf19b:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:44:36,835 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,836 DEBUG [RS:1;f64cf8bdf19b:42865 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:44:36,836 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,836 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,836 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,42741,1732779876434-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T07:44:36,836 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,836 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,836 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,836 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,836 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,837 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,42865,1732779876461-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T07:44:36,858 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T07:44:36,858 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,42741,1732779876434-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,858 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,858 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.Replication(171): f64cf8bdf19b,42741,1732779876434 started 2024-11-28T07:44:36,862 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T07:44:36,862 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,42865,1732779876461-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,862 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,862 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.Replication(171): f64cf8bdf19b,42865,1732779876461 started 2024-11-28T07:44:36,877 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,878 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer(1482): Serving as f64cf8bdf19b,42741,1732779876434, RpcServer on f64cf8bdf19b/172.17.0.2:42741, sessionid=0x1015d888a9d0001 2024-11-28T07:44:36,878 DEBUG [RS:0;f64cf8bdf19b:42741 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T07:44:36,878 DEBUG [RS:0;f64cf8bdf19b:42741 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f64cf8bdf19b,42741,1732779876434 2024-11-28T07:44:36,878 DEBUG [RS:0;f64cf8bdf19b:42741 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f64cf8bdf19b,42741,1732779876434' 2024-11-28T07:44:36,878 DEBUG [RS:0;f64cf8bdf19b:42741 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T07:44:36,879 DEBUG [RS:0;f64cf8bdf19b:42741 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T07:44:36,879 DEBUG [RS:0;f64cf8bdf19b:42741 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T07:44:36,879 DEBUG [RS:0;f64cf8bdf19b:42741 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T07:44:36,879 DEBUG [RS:0;f64cf8bdf19b:42741 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f64cf8bdf19b,42741,1732779876434 2024-11-28T07:44:36,879 DEBUG [RS:0;f64cf8bdf19b:42741 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f64cf8bdf19b,42741,1732779876434' 2024-11-28T07:44:36,879 DEBUG [RS:0;f64cf8bdf19b:42741 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T07:44:36,880 DEBUG [RS:0;f64cf8bdf19b:42741 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T07:44:36,880 DEBUG [RS:0;f64cf8bdf19b:42741 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T07:44:36,880 INFO [RS:0;f64cf8bdf19b:42741 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T07:44:36,880 INFO [RS:0;f64cf8bdf19b:42741 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T07:44:36,881 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:36,881 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.HRegionServer(1482): Serving as f64cf8bdf19b,42865,1732779876461, RpcServer on f64cf8bdf19b/172.17.0.2:42865, sessionid=0x1015d888a9d0002 2024-11-28T07:44:36,881 DEBUG [RS:1;f64cf8bdf19b:42865 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T07:44:36,881 DEBUG [RS:1;f64cf8bdf19b:42865 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f64cf8bdf19b,42865,1732779876461 2024-11-28T07:44:36,881 DEBUG [RS:1;f64cf8bdf19b:42865 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f64cf8bdf19b,42865,1732779876461' 2024-11-28T07:44:36,881 DEBUG [RS:1;f64cf8bdf19b:42865 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T07:44:36,881 DEBUG [RS:1;f64cf8bdf19b:42865 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T07:44:36,882 DEBUG [RS:1;f64cf8bdf19b:42865 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T07:44:36,882 DEBUG [RS:1;f64cf8bdf19b:42865 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T07:44:36,882 DEBUG [RS:1;f64cf8bdf19b:42865 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f64cf8bdf19b,42865,1732779876461 2024-11-28T07:44:36,882 DEBUG [RS:1;f64cf8bdf19b:42865 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f64cf8bdf19b,42865,1732779876461' 2024-11-28T07:44:36,882 DEBUG [RS:1;f64cf8bdf19b:42865 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T07:44:36,882 DEBUG [RS:1;f64cf8bdf19b:42865 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T07:44:36,883 DEBUG [RS:1;f64cf8bdf19b:42865 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T07:44:36,883 INFO [RS:1;f64cf8bdf19b:42865 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T07:44:36,883 INFO [RS:1;f64cf8bdf19b:42865 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T07:44:36,912 WARN [f64cf8bdf19b:43047 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T07:44:36,981 INFO [RS:0;f64cf8bdf19b:42741 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T07:44:36,982 INFO [RS:0;f64cf8bdf19b:42741 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f64cf8bdf19b%2C42741%2C1732779876434, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,42741,1732779876434, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/oldWALs, maxLogs=55 2024-11-28T07:44:36,983 INFO [RS:1;f64cf8bdf19b:42865 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T07:44:36,984 INFO [RS:1;f64cf8bdf19b:42865 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f64cf8bdf19b%2C42865%2C1732779876461, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,42865,1732779876461, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/oldWALs, maxLogs=55 2024-11-28T07:44:37,002 INFO [RS:0;f64cf8bdf19b:42741 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,42741,1732779876434/f64cf8bdf19b%2C42741%2C1732779876434.1732779876983 2024-11-28T07:44:37,003 DEBUG [RS:0;f64cf8bdf19b:42741 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T07:44:37,005 INFO [RS:1;f64cf8bdf19b:42865 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,42865,1732779876461/f64cf8bdf19b%2C42865%2C1732779876461.1732779876986 2024-11-28T07:44:37,006 DEBUG [RS:1;f64cf8bdf19b:42865 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T07:44:37,162 DEBUG [f64cf8bdf19b:43047 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-28T07:44:37,163 DEBUG [f64cf8bdf19b:43047 {}] balancer.BalancerClusterState(204): Hosts are {f64cf8bdf19b=0} racks are {/default-rack=0} 2024-11-28T07:44:37,164 DEBUG [f64cf8bdf19b:43047 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T07:44:37,164 DEBUG [f64cf8bdf19b:43047 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T07:44:37,164 DEBUG [f64cf8bdf19b:43047 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T07:44:37,164 DEBUG [f64cf8bdf19b:43047 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T07:44:37,164 INFO [f64cf8bdf19b:43047 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T07:44:37,164 INFO [f64cf8bdf19b:43047 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T07:44:37,164 DEBUG [f64cf8bdf19b:43047 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T07:44:37,164 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f64cf8bdf19b,42741,1732779876434 2024-11-28T07:44:37,165 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f64cf8bdf19b,42741,1732779876434, state=OPENING 2024-11-28T07:44:37,167 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-28T07:44:37,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:37,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:37,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:37,169 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:37,169 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:37,169 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:37,169 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:44:37,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f64cf8bdf19b,42741,1732779876434}] 2024-11-28T07:44:37,322 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T07:44:37,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40265, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T07:44:37,326 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T07:44:37,326 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:44:37,327 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T07:44:37,328 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f64cf8bdf19b%2C42741%2C1732779876434.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,42741,1732779876434, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/oldWALs, maxLogs=55 2024-11-28T07:44:37,346 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,42741,1732779876434/f64cf8bdf19b%2C42741%2C1732779876434.meta.1732779877328.meta 2024-11-28T07:44:37,346 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T07:44:37,346 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:44:37,347 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T07:44:37,347 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T07:44:37,347 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T07:44:37,347 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T07:44:37,347 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:44:37,347 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T07:44:37,347 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T07:44:37,348 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-28T07:44:37,349 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T07:44:37,349 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:37,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:37,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-28T07:44:37,350 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T07:44:37,350 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:37,350 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:37,350 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-28T07:44:37,350 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T07:44:37,351 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:37,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:37,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-28T07:44:37,351 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T07:44:37,351 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:37,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:37,352 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T07:44:37,352 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740 2024-11-28T07:44:37,352 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740 2024-11-28T07:44:37,353 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T07:44:37,353 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T07:44:37,353 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T07:44:37,354 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T07:44:37,354 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66878679, jitterRate=-0.0034300237894058228}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T07:44:37,354 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T07:44:37,355 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732779877347Writing region info on filesystem at 1732779877347Initializing all the Stores at 1732779877348 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779877348Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779877348Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779877348Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779877348Cleaning up temporary data from old regions at 1732779877353 (+5 ms)Running coprocessor post-open hooks at 1732779877354 (+1 ms)Region opened successfully at 1732779877355 (+1 ms) 2024-11-28T07:44:37,355 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732779877322 2024-11-28T07:44:37,358 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T07:44:37,358 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T07:44:37,358 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f64cf8bdf19b,42741,1732779876434 2024-11-28T07:44:37,359 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f64cf8bdf19b,42741,1732779876434, state=OPEN 2024-11-28T07:44:37,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:44:37,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:44:37,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:44:37,362 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f64cf8bdf19b,42741,1732779876434 2024-11-28T07:44:37,362 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:37,362 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:37,362 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:37,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-28T07:44:37,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f64cf8bdf19b,42741,1732779876434 in 193 msec 2024-11-28T07:44:37,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-28T07:44:37,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 605 msec 2024-11-28T07:44:37,368 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T07:44:37,368 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-28T07:44:37,371 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:44:37,371 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,42741,1732779876434, seqNum=-1] 2024-11-28T07:44:37,371 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:44:37,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41541, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:44:37,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 674 msec 2024-11-28T07:44:37,378 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732779877378, completionTime=-1 2024-11-28T07:44:37,378 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-28T07:44:37,378 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:44:37,380 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T07:44:37,380 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732779937380 2024-11-28T07:44:37,380 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732779997380 2024-11-28T07:44:37,380 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-28T07:44:37,381 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,43047,1732779876381-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:37,381 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,43047,1732779876381-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:37,381 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,43047,1732779876381-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:37,381 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f64cf8bdf19b:43047, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:37,381 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:37,381 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:37,384 DEBUG [master/f64cf8bdf19b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-28T07:44:37,386 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.875sec 2024-11-28T07:44:37,386 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-28T07:44:37,386 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-28T07:44:37,386 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-28T07:44:37,386 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-28T07:44:37,386 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-28T07:44:37,386 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,43047,1732779876381-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T07:44:37,386 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,43047,1732779876381-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-28T07:44:37,387 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-28T07:44:37,387 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-28T07:44:37,387 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,43047,1732779876381-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:37,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc06048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:44:37,483 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,43047,-1 for getting cluster id 2024-11-28T07:44:37,483 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:44:37,485 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '47067c86-19cb-49df-9194-087e1d2051ef' 2024-11-28T07:44:37,485 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:44:37,485 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "47067c86-19cb-49df-9194-087e1d2051ef" 2024-11-28T07:44:37,486 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@748b46aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:44:37,486 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,43047,-1] 2024-11-28T07:44:37,486 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:44:37,486 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:37,488 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:44:37,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:44:37,489 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:44:37,490 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,42741,1732779876434, seqNum=-1] 2024-11-28T07:44:37,491 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:44:37,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:44:37,496 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f64cf8bdf19b,43047,1732779876381 2024-11-28T07:44:37,496 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T07:44:37,497 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f64cf8bdf19b,43047,1732779876381 2024-11-28T07:44:37,497 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@754d162c 2024-11-28T07:44:37,497 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T07:44:37,499 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T07:44:37,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43047 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732779877496', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T07:44:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43047 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732779877496 2024-11-28T07:44:37,502 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732779877496 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T07:44:37,502 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:37,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43047 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732779877496" procId is: 4 2024-11-28T07:44:37,503 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732779877496 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T07:44:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:44:37,514 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732779877496-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4c22f09a39e6b82eb676be3e58313ff1, NAME => 'testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732779877496', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597 2024-11-28T07:44:37,521 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732779877496-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:44:37,521 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732779877496-pool-0 {}] regionserver.HRegion(1722): Closing 4c22f09a39e6b82eb676be3e58313ff1, disabling compactions & flushes 2024-11-28T07:44:37,521 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732779877496-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:37,521 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732779877496-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:37,521 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732779877496-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. after waiting 0 ms 2024-11-28T07:44:37,521 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732779877496-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:37,522 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732779877496-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:37,522 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732779877496-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4c22f09a39e6b82eb676be3e58313ff1: Waiting for close lock at 1732779877521Disabling compacts and flushes for region at 1732779877521Disabling writes for close at 1732779877521Writing region close event to WAL at 1732779877522 (+1 ms)Closed at 1732779877522 2024-11-28T07:44:37,523 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732779877496 execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T07:44:37,523 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732779877523"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732779877523"}]},"ts":"1732779877523"} 2024-11-28T07:44:37,525 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T07:44:37,526 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732779877496 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T07:44:37,526 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732779877496","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779877526"}]},"ts":"1732779877526"} 2024-11-28T07:44:37,527 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732779877496, state=ENABLING in hbase:meta 2024-11-28T07:44:37,528 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {f64cf8bdf19b=0} racks are {/default-rack=0} 2024-11-28T07:44:37,528 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T07:44:37,528 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T07:44:37,528 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T07:44:37,528 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T07:44:37,528 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T07:44:37,528 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T07:44:37,528 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T07:44:37,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732779877496, region=4c22f09a39e6b82eb676be3e58313ff1, ASSIGN}] 2024-11-28T07:44:37,529 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732779877496, region=4c22f09a39e6b82eb676be3e58313ff1, ASSIGN 2024-11-28T07:44:37,530 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732779877496, region=4c22f09a39e6b82eb676be3e58313ff1, ASSIGN; state=OFFLINE, location=f64cf8bdf19b,42741,1732779876434; forceNewPlan=false, retain=false 2024-11-28T07:44:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:44:37,681 INFO [f64cf8bdf19b:43047 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-28T07:44:37,681 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4c22f09a39e6b82eb676be3e58313ff1, regionState=OPENING, regionLocation=f64cf8bdf19b,42741,1732779876434 2024-11-28T07:44:37,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732779877496, region=4c22f09a39e6b82eb676be3e58313ff1, ASSIGN because future has completed 2024-11-28T07:44:37,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c22f09a39e6b82eb676be3e58313ff1, server=f64cf8bdf19b,42741,1732779876434}] 2024-11-28T07:44:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:44:37,839 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:37,839 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c22f09a39e6b82eb676be3e58313ff1, NAME => 'testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:44:37,839 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732779877496 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:37,840 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:44:37,840 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:37,840 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:37,841 INFO [StoreOpener-4c22f09a39e6b82eb676be3e58313ff1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:37,845 INFO [StoreOpener-4c22f09a39e6b82eb676be3e58313ff1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c22f09a39e6b82eb676be3e58313ff1 columnFamilyName family 2024-11-28T07:44:37,845 DEBUG [StoreOpener-4c22f09a39e6b82eb676be3e58313ff1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:37,845 INFO [StoreOpener-4c22f09a39e6b82eb676be3e58313ff1-1 {}] regionserver.HStore(327): Store=4c22f09a39e6b82eb676be3e58313ff1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:44:37,845 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:37,846 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:37,846 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:37,846 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:37,846 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:37,847 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:37,853 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:44:37,853 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4c22f09a39e6b82eb676be3e58313ff1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69021826, jitterRate=0.0285053551197052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:44:37,853 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:37,854 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4c22f09a39e6b82eb676be3e58313ff1: Running coprocessor pre-open hook at 1732779877840Writing region info on filesystem at 1732779877840Initializing all the Stores at 1732779877840Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779877840Cleaning up temporary data from old regions at 1732779877846 (+6 ms)Running coprocessor post-open hooks at 1732779877853 (+7 ms)Region opened successfully at 1732779877854 (+1 ms) 2024-11-28T07:44:37,855 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1., pid=6, masterSystemTime=1732779877837 2024-11-28T07:44:37,857 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:37,857 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:37,858 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4c22f09a39e6b82eb676be3e58313ff1, regionState=OPEN, openSeqNum=2, regionLocation=f64cf8bdf19b,42741,1732779876434 2024-11-28T07:44:37,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c22f09a39e6b82eb676be3e58313ff1, server=f64cf8bdf19b,42741,1732779876434 because future has completed 2024-11-28T07:44:37,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-28T07:44:37,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c22f09a39e6b82eb676be3e58313ff1, server=f64cf8bdf19b,42741,1732779876434 in 177 msec 2024-11-28T07:44:37,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-28T07:44:37,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732779877496, region=4c22f09a39e6b82eb676be3e58313ff1, ASSIGN in 334 msec 2024-11-28T07:44:37,865 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732779877496 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T07:44:37,866 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732779877496","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779877866"}]},"ts":"1732779877866"} 2024-11-28T07:44:37,868 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732779877496, state=ENABLED in hbase:meta 2024-11-28T07:44:37,868 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732779877496 execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T07:44:37,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732779877496 in 369 msec 2024-11-28T07:44:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43047 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:44:38,134 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732779877496 completed 2024-11-28T07:44:38,134 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732779877496 get assigned. Timeout = 60000ms 2024-11-28T07:44:38,134 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T07:44:38,137 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732779877496 assigned to meta. Checking AM states. 2024-11-28T07:44:38,138 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T07:44:38,138 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732779877496 assigned. 2024-11-28T07:44:38,141 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732779877496', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1., hostname=f64cf8bdf19b,42741,1732779876434, seqNum=2] 2024-11-28T07:44:38,145 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4c22f09a39e6b82eb676be3e58313ff1 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-28T07:44:38,155 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cfebb5664b9f41178261084200076d26 2024-11-28T07:44:38,169 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/.tmp/family/cfebb5664b9f41178261084200076d26 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-28T07:44:38,169 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/.tmp/family/cfebb5664b9f41178261084200076d26 2024-11-28T07:44:38,172 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/.tmp/family/cfebb5664b9f41178261084200076d26 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/family/cfebb5664b9f41178261084200076d26 2024-11-28T07:44:38,173 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/family/cfebb5664b9f41178261084200076d26, delay=927 ms 2024-11-28T07:44:38,174 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/family/cfebb5664b9f41178261084200076d26, entries=2, sequenceid=6, filesize=4.9 K 2024-11-28T07:44:38,175 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 4c22f09a39e6b82eb676be3e58313ff1 in 30ms, sequenceid=6, compaction requested=false 2024-11-28T07:44:38,175 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732779877496' 2024-11-28T07:44:38,175 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4c22f09a39e6b82eb676be3e58313ff1: 2024-11-28T07:44:38,703 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T07:44:39,101 DEBUG [hfile-prefetch-1732779875820 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key cfebb5664b9f41178261084200076d26_0. 2024-11-28T07:44:39,102 DEBUG [hfile-prefetch-1732779875820 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key cfebb5664b9f41178261084200076d26_117. 2024-11-28T07:44:39,102 DEBUG [hfile-prefetch-1732779875820 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/family/cfebb5664b9f41178261084200076d26, with total blocks 2, and data blocks 1 2024-11-28T07:44:39,102 DEBUG [hfile-prefetch-1732779875820 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-28T07:44:39,102 DEBUG [hfile-prefetch-1732779875820 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/family/cfebb5664b9f41178261084200076d26, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-28T07:44:39,102 DEBUG [hfile-prefetch-1732779875820 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/family/cfebb5664b9f41178261084200076d26. Will sleep 100ms and try the verification again. 2024-11-28T07:44:39,144 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-28T07:44:39,144 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-28T07:44:39,144 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T07:44:39,144 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T07:44:39,152 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-28T07:44:39,152 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732779879144 2024-11-28T07:44:39,152 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-28T07:44:39,176 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;f64cf8bdf19b:42741,5,FailOnTimeoutGroup] 2024-11-28T07:44:39,176 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f64cf8bdf19b,42741,1732779876434' ***** 2024-11-28T07:44:39,176 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-28T07:44:39,176 INFO [RS:0;f64cf8bdf19b:42741 {}] bucket.BucketCache(1738): Disabling cache 2024-11-28T07:44:39,176 INFO [RS:0;f64cf8bdf19b:42741 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/bucket.persistence 2024-11-28T07:44:39,176 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-28T07:44:39,176 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-28T07:44:39,176 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-28T07:44:39,177 INFO [RS:0;f64cf8bdf19b:42741 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-28T07:44:39,177 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-28T07:44:39,187 DEBUG [RS:0;f64cf8bdf19b:42741 {}] bucket.BucketCache(1487): Thread RS:0;f64cf8bdf19b:42741 started persisting bucket cache to file 2024-11-28T07:44:39,187 DEBUG [RS:0;f64cf8bdf19b:42741 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T07:44:39,187 DEBUG [RS:0;f64cf8bdf19b:42741 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T07:44:39,190 DEBUG [RS:0;f64cf8bdf19b:42741 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-28T07:44:39,190 DEBUG [RS:0;f64cf8bdf19b:42741 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732779879187 2024-11-28T07:44:39,190 DEBUG [RS:0;f64cf8bdf19b:42741 {}] bucket.BucketCache(1509): Thread RS:0;f64cf8bdf19b:42741 finished persisting bucket cache to file, renaming 2024-11-28T07:44:39,190 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T07:44:39,190 INFO [RS:0;f64cf8bdf19b:42741 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T07:44:39,190 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T07:44:39,190 INFO [RS:0;f64cf8bdf19b:42741 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T07:44:39,191 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer(3091): Received CLOSE for 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:39,191 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer(959): stopping server f64cf8bdf19b,42741,1732779876434 2024-11-28T07:44:39,191 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T07:44:39,191 INFO [RS:0;f64cf8bdf19b:42741 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f64cf8bdf19b:42741. 2024-11-28T07:44:39,191 DEBUG [RS:0;f64cf8bdf19b:42741 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:44:39,191 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T07:44:39,191 DEBUG [RS:0;f64cf8bdf19b:42741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:39,191 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4c22f09a39e6b82eb676be3e58313ff1, disabling compactions & flushes 2024-11-28T07:44:39,191 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T07:44:39,191 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T07:44:39,191 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T07:44:39,191 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:39,191 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T07:44:39,191 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:39,191 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. after waiting 0 ms 2024-11-28T07:44:39,191 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:39,192 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-28T07:44:39,192 DEBUG [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 4c22f09a39e6b82eb676be3e58313ff1=testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1.} 2024-11-28T07:44:39,192 DEBUG [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:39,192 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T07:44:39,192 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T07:44:39,192 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T07:44:39,192 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T07:44:39,192 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T07:44:39,192 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T07:44:39,192 DEBUG [StoreFileCloser-4c22f09a39e6b82eb676be3e58313ff1-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/family/cfebb5664b9f41178261084200076d26 2024-11-28T07:44:39,192 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-28T07:44:39,192 WARN [hfile-prefetch-1732779875820 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/family/cfebb5664b9f41178261084200076d26, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-28T07:44:39,193 DEBUG [hfile-prefetch-1732779875820 {}] hfile.PrefetchExecutor(121): Prefetch completed for cfebb5664b9f41178261084200076d26 2024-11-28T07:44:39,197 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-28T07:44:39,197 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:39,197 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4c22f09a39e6b82eb676be3e58313ff1: Waiting for close lock at 1732779879191Running coprocessor pre-close hooks at 1732779879191Disabling compacts and flushes for region at 1732779879191Disabling writes for close at 1732779879191Writing region close event to WAL at 1732779879192 (+1 ms)Running coprocessor post-close hooks at 1732779879197 (+5 ms)Closed at 1732779879197 2024-11-28T07:44:39,197 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:39,202 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4bf69875c4864968b4c68b0d76a94382 2024-11-28T07:44:39,216 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/info/4bf69875c4864968b4c68b0d76a94382 is 209, key is testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1./info:regioninfo/1732779877858/Put/seqid=0 2024-11-28T07:44:39,217 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/info/4bf69875c4864968b4c68b0d76a94382 2024-11-28T07:44:39,225 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c523d09e42e64248aa9202a094f5287a 2024-11-28T07:44:39,238 INFO [regionserver/f64cf8bdf19b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:44:39,238 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/ns/c523d09e42e64248aa9202a094f5287a is 43, key is default/ns:d/1732779877374/Put/seqid=0 2024-11-28T07:44:39,239 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/ns/c523d09e42e64248aa9202a094f5287a 2024-11-28T07:44:39,247 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2d417ac8343544ef8ae37b4aca43ddbd 2024-11-28T07:44:39,260 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/table/2d417ac8343544ef8ae37b4aca43ddbd is 80, key is testBlockEvictionOnGracefulStop1732779877496/table:state/1732779877866/Put/seqid=0 2024-11-28T07:44:39,261 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/table/2d417ac8343544ef8ae37b4aca43ddbd 2024-11-28T07:44:39,263 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/info/4bf69875c4864968b4c68b0d76a94382 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/info/4bf69875c4864968b4c68b0d76a94382 2024-11-28T07:44:39,265 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/info/4bf69875c4864968b4c68b0d76a94382, delay=900 ms 2024-11-28T07:44:39,266 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/info/4bf69875c4864968b4c68b0d76a94382, entries=10, sequenceid=11, filesize=7.0 K 2024-11-28T07:44:39,266 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/ns/c523d09e42e64248aa9202a094f5287a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/ns/c523d09e42e64248aa9202a094f5287a 2024-11-28T07:44:39,267 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/ns/c523d09e42e64248aa9202a094f5287a, delay=947 ms 2024-11-28T07:44:39,268 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/ns/c523d09e42e64248aa9202a094f5287a, entries=2, sequenceid=11, filesize=5.0 K 2024-11-28T07:44:39,268 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/table/2d417ac8343544ef8ae37b4aca43ddbd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/table/2d417ac8343544ef8ae37b4aca43ddbd 2024-11-28T07:44:39,270 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/table/2d417ac8343544ef8ae37b4aca43ddbd, delay=955 ms 2024-11-28T07:44:39,270 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/table/2d417ac8343544ef8ae37b4aca43ddbd, entries=2, sequenceid=11, filesize=5.3 K 2024-11-28T07:44:39,271 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=11, compaction requested=false 2024-11-28T07:44:39,272 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/info/4bf69875c4864968b4c68b0d76a94382 2024-11-28T07:44:39,273 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/ns/c523d09e42e64248aa9202a094f5287a 2024-11-28T07:44:39,274 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/table/2d417ac8343544ef8ae37b4aca43ddbd 2024-11-28T07:44:39,279 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-28T07:44:39,280 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T07:44:39,280 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T07:44:39,280 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732779879192Running coprocessor pre-close hooks at 1732779879192Disabling compacts and flushes for region at 1732779879192Disabling writes for close at 1732779879192Obtaining lock to block concurrent updates at 1732779879192Preparing flush snapshotting stores in 1588230740 at 1732779879192Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732779879193 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732779879193Flushing 1588230740/info: creating writer at 1732779879193Flushing 1588230740/info: appending metadata at 1732779879216 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732779879216Flushing 1588230740/ns: creating writer at 1732779879219 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732779879238 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732779879238Flushing 1588230740/table: creating writer at 1732779879241 (+3 ms)Flushing 1588230740/table: appending metadata at 1732779879260 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732779879260Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b8b60b6: reopening flushed file at 1732779879263 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74c79bd3: reopening flushed file at 1732779879266 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ceafc64: reopening flushed file at 1732779879268 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=11, compaction requested=false at 1732779879271 (+3 ms)Writing region close event to WAL at 1732779879274 (+3 ms)Running coprocessor post-close hooks at 1732779879280 (+6 ms)Closed at 1732779879280 2024-11-28T07:44:39,280 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T07:44:39,392 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer(976): stopping server f64cf8bdf19b,42741,1732779876434; all regions closed. 2024-11-28T07:44:39,394 DEBUG [RS:0;f64cf8bdf19b:42741 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/oldWALs 2024-11-28T07:44:39,394 INFO [RS:0;f64cf8bdf19b:42741 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f64cf8bdf19b%2C42741%2C1732779876434.meta:.meta(num 1732779877328) 2024-11-28T07:44:39,395 DEBUG [RS:0;f64cf8bdf19b:42741 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/oldWALs 2024-11-28T07:44:39,395 INFO [RS:0;f64cf8bdf19b:42741 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f64cf8bdf19b%2C42741%2C1732779876434:(num 1732779876983) 2024-11-28T07:44:39,395 DEBUG [RS:0;f64cf8bdf19b:42741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:39,395 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:44:39,396 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T07:44:39,396 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.ChoreService(370): Chore service for: regionserver/f64cf8bdf19b:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-28T07:44:39,396 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T07:44:39,396 INFO [regionserver/f64cf8bdf19b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T07:44:39,396 INFO [RS:0;f64cf8bdf19b:42741 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42741 2024-11-28T07:44:39,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f64cf8bdf19b,42741,1732779876434 2024-11-28T07:44:39,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T07:44:39,401 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T07:44:39,402 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f64cf8bdf19b,42741,1732779876434] 2024-11-28T07:44:39,404 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f64cf8bdf19b,42741,1732779876434 already deleted, retry=false 2024-11-28T07:44:39,404 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of f64cf8bdf19b,42741,1732779876434 on f64cf8bdf19b,43047,1732779876381 2024-11-28T07:44:39,409 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure f64cf8bdf19b,42741,1732779876434, splitWal=true, meta=true 2024-11-28T07:44:39,411 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure f64cf8bdf19b,42741,1732779876434, splitWal=true, meta=true 2024-11-28T07:44:39,411 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for f64cf8bdf19b,42741,1732779876434 (carryingMeta=true) f64cf8bdf19b,42741,1732779876434/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@7babb63b[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-28T07:44:39,412 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure f64cf8bdf19b,42741,1732779876434, splitWal=true, meta=true, isMeta: true 2024-11-28T07:44:39,413 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,42741,1732779876434-splitting 2024-11-28T07:44:39,413 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,42741,1732779876434-splitting dir is empty, no logs to split. 2024-11-28T07:44:39,413 INFO [PEWorker-2 {}] master.SplitWALManager(105): f64cf8bdf19b,42741,1732779876434 WAL count=0, meta=true 2024-11-28T07:44:39,415 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,42741,1732779876434-splitting dir is empty, no logs to split. 2024-11-28T07:44:39,415 INFO [PEWorker-2 {}] master.SplitWALManager(105): f64cf8bdf19b,42741,1732779876434 WAL count=0, meta=true 2024-11-28T07:44:39,415 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if f64cf8bdf19b,42741,1732779876434 WAL splitting is done? wals=0, meta=true 2024-11-28T07:44:39,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T07:44:39,417 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:44:39,418 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-28T07:44:39,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:44:39,503 INFO [RS:0;f64cf8bdf19b:42741 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T07:44:39,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42741-0x1015d888a9d0001, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:44:39,504 INFO [RS:0;f64cf8bdf19b:42741 {}] regionserver.HRegionServer(1031): Exiting; stopping=f64cf8bdf19b,42741,1732779876434; zookeeper connection closed. 2024-11-28T07:44:39,504 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5f6f8de4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5f6f8de4 2024-11-28T07:44:39,568 DEBUG [f64cf8bdf19b:43047 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-28T07:44:39,568 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f64cf8bdf19b,42865,1732779876461 2024-11-28T07:44:39,569 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f64cf8bdf19b,42865,1732779876461, state=OPENING 2024-11-28T07:44:39,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:44:39,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:44:39,571 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:39,571 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:44:39,571 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:39,571 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f64cf8bdf19b,42865,1732779876461}] 2024-11-28T07:44:39,693 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f64cf8bdf19b:0 server-side Connection retries=45 2024-11-28T07:44:39,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:39,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:39,693 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T07:44:39,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:44:39,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T07:44:39,693 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T07:44:39,694 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T07:44:39,694 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44077 2024-11-28T07:44:39,696 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44077 connecting to ZooKeeper ensemble=127.0.0.1:64536 2024-11-28T07:44:39,696 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:39,696 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:39,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:440770x0, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T07:44:39,699 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44077-0x1015d888a9d0003 connected 2024-11-28T07:44:39,699 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44077-0x1015d888a9d0003, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-28T07:44:39,699 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-28T07:44:39,700 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T07:44:39,701 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/bucket.cache 2024-11-28T07:44:39,701 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-28T07:44:39,701 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-28T07:44:39,701 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-28T07:44:39,701 INFO [Thread-206 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-28T07:44:39,701 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-28T07:44:39,702 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T07:44:39,702 INFO [Thread-206 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-28T07:44:39,702 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44077-0x1015d888a9d0003, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T07:44:39,704 INFO [Thread-206 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-28T07:44:39,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44077-0x1015d888a9d0003, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T07:44:39,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44077 2024-11-28T07:44:39,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44077 2024-11-28T07:44:39,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44077 2024-11-28T07:44:39,707 DEBUG [Thread-206 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-28T07:44:39,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44077 2024-11-28T07:44:39,707 INFO [Thread-206 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-28T07:44:39,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44077 2024-11-28T07:44:39,707 INFO [Thread-206 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-28T07:44:39,707 INFO [Thread-206 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/bucket.persistence is complete. 2024-11-28T07:44:39,709 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.HRegionServer(746): ClusterId : 47067c86-19cb-49df-9194-087e1d2051ef 2024-11-28T07:44:39,709 DEBUG [RS:2;f64cf8bdf19b:44077 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T07:44:39,712 DEBUG [RS:2;f64cf8bdf19b:44077 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T07:44:39,712 DEBUG [RS:2;f64cf8bdf19b:44077 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T07:44:39,714 DEBUG [RS:2;f64cf8bdf19b:44077 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T07:44:39,714 DEBUG [RS:2;f64cf8bdf19b:44077 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c3624f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f64cf8bdf19b/172.17.0.2:0 2024-11-28T07:44:39,724 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T07:44:39,726 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38261, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T07:44:39,728 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T07:44:39,728 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:44:39,728 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T07:44:39,729 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f64cf8bdf19b%2C42865%2C1732779876461.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,42865,1732779876461, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/oldWALs, maxLogs=55 2024-11-28T07:44:39,729 DEBUG [RS:2;f64cf8bdf19b:44077 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;f64cf8bdf19b:44077 2024-11-28T07:44:39,729 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T07:44:39,729 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T07:44:39,729 DEBUG [RS:2;f64cf8bdf19b:44077 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T07:44:39,730 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.HRegionServer(2659): reportForDuty to master=f64cf8bdf19b,43047,1732779876381 with port=44077, startcode=1732779879693 2024-11-28T07:44:39,730 DEBUG [RS:2;f64cf8bdf19b:44077 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T07:44:39,732 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34653, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T07:44:39,732 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43047 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f64cf8bdf19b,44077,1732779879693 2024-11-28T07:44:39,732 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43047 {}] master.ServerManager(517): Registering regionserver=f64cf8bdf19b,44077,1732779879693 2024-11-28T07:44:39,734 DEBUG [RS:2;f64cf8bdf19b:44077 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597 2024-11-28T07:44:39,734 DEBUG [RS:2;f64cf8bdf19b:44077 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-28T07:44:39,734 DEBUG [RS:2;f64cf8bdf19b:44077 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T07:44:39,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T07:44:39,736 DEBUG [RS:2;f64cf8bdf19b:44077 {}] zookeeper.ZKUtil(111): regionserver:44077-0x1015d888a9d0003, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f64cf8bdf19b,44077,1732779879693 2024-11-28T07:44:39,736 WARN [RS:2;f64cf8bdf19b:44077 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T07:44:39,736 INFO [RS:2;f64cf8bdf19b:44077 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:44:39,736 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f64cf8bdf19b,44077,1732779879693] 2024-11-28T07:44:39,736 DEBUG [RS:2;f64cf8bdf19b:44077 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,44077,1732779879693 2024-11-28T07:44:39,745 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T07:44:39,747 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T07:44:39,748 INFO [RS:2;f64cf8bdf19b:44077 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T07:44:39,748 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:39,748 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T07:44:39,748 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T07:44:39,748 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:39,749 DEBUG [RS:2;f64cf8bdf19b:44077 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:39,749 DEBUG [RS:2;f64cf8bdf19b:44077 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:39,749 DEBUG [RS:2;f64cf8bdf19b:44077 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:39,749 DEBUG [RS:2;f64cf8bdf19b:44077 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:39,749 DEBUG [RS:2;f64cf8bdf19b:44077 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:39,749 DEBUG [RS:2;f64cf8bdf19b:44077 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f64cf8bdf19b:0, corePoolSize=2, maxPoolSize=2 2024-11-28T07:44:39,749 DEBUG [RS:2;f64cf8bdf19b:44077 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:39,749 DEBUG [RS:2;f64cf8bdf19b:44077 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:39,749 DEBUG [RS:2;f64cf8bdf19b:44077 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:39,749 DEBUG [RS:2;f64cf8bdf19b:44077 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:39,749 DEBUG [RS:2;f64cf8bdf19b:44077 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:39,749 DEBUG [RS:2;f64cf8bdf19b:44077 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:44:39,749 DEBUG [RS:2;f64cf8bdf19b:44077 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f64cf8bdf19b:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:44:39,749 DEBUG [RS:2;f64cf8bdf19b:44077 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:44:39,750 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:39,750 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:39,750 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:39,750 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:39,750 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:39,750 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,44077,1732779879693-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T07:44:39,756 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,42865,1732779876461/f64cf8bdf19b%2C42865%2C1732779876461.meta.1732779879730.meta 2024-11-28T07:44:39,761 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T07:44:39,762 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:44:39,762 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T07:44:39,762 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T07:44:39,763 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T07:44:39,763 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T07:44:39,763 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:44:39,763 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T07:44:39,763 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T07:44:39,764 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-28T07:44:39,765 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T07:44:39,765 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:39,767 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/info/4bf69875c4864968b4c68b0d76a94382, delay=980 ms 2024-11-28T07:44:39,768 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T07:44:39,768 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,44077,1732779879693-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:39,768 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:39,768 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.Replication(171): f64cf8bdf19b,44077,1732779879693 started 2024-11-28T07:44:39,769 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/info/4bf69875c4864968b4c68b0d76a94382 2024-11-28T07:44:39,769 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:39,769 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-28T07:44:39,770 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T07:44:39,770 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:39,772 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/ns/c523d09e42e64248aa9202a094f5287a, delay=960 ms 2024-11-28T07:44:39,773 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/ns/c523d09e42e64248aa9202a094f5287a 2024-11-28T07:44:39,773 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:39,774 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-28T07:44:39,774 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T07:44:39,774 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:39,774 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:39,774 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-28T07:44:39,775 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T07:44:39,775 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:39,777 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/table/2d417ac8343544ef8ae37b4aca43ddbd, delay=983 ms 2024-11-28T07:44:39,778 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/table/2d417ac8343544ef8ae37b4aca43ddbd 2024-11-28T07:44:39,778 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:44:39,778 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T07:44:39,779 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740 2024-11-28T07:44:39,779 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740 2024-11-28T07:44:39,779 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T07:44:39,779 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T07:44:39,780 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T07:44:39,781 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T07:44:39,781 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59334917, jitterRate=-0.1158408373594284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T07:44:39,781 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T07:44:39,782 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732779879763Writing region info on filesystem at 1732779879763Initializing all the Stores at 1732779879764 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779879764Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779879764Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779879764Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779879764Cleaning up temporary data from old regions at 1732779879779 (+15 ms)Running coprocessor post-open hooks at 1732779879781 (+2 ms)Region opened successfully at 1732779879782 (+1 ms) 2024-11-28T07:44:39,783 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732779879723 2024-11-28T07:44:39,786 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T07:44:39,786 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T07:44:39,786 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=f64cf8bdf19b,42865,1732779876461 2024-11-28T07:44:39,787 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f64cf8bdf19b,42865,1732779876461, state=OPEN 2024-11-28T07:44:39,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:44:39,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44077-0x1015d888a9d0003, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:44:39,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:44:39,789 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f64cf8bdf19b,42865,1732779876461 2024-11-28T07:44:39,789 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:39,789 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:39,789 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:44:39,790 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:44:39,790 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.HRegionServer(1482): Serving as f64cf8bdf19b,44077,1732779879693, RpcServer on f64cf8bdf19b/172.17.0.2:44077, sessionid=0x1015d888a9d0003 2024-11-28T07:44:39,790 DEBUG [RS:2;f64cf8bdf19b:44077 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T07:44:39,790 DEBUG [RS:2;f64cf8bdf19b:44077 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f64cf8bdf19b,44077,1732779879693 2024-11-28T07:44:39,790 DEBUG [RS:2;f64cf8bdf19b:44077 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f64cf8bdf19b,44077,1732779879693' 2024-11-28T07:44:39,790 DEBUG [RS:2;f64cf8bdf19b:44077 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T07:44:39,791 DEBUG [RS:2;f64cf8bdf19b:44077 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T07:44:39,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-28T07:44:39,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f64cf8bdf19b,42865,1732779876461 in 218 msec 2024-11-28T07:44:39,792 DEBUG [RS:2;f64cf8bdf19b:44077 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T07:44:39,792 DEBUG [RS:2;f64cf8bdf19b:44077 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T07:44:39,792 DEBUG [RS:2;f64cf8bdf19b:44077 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f64cf8bdf19b,44077,1732779879693 2024-11-28T07:44:39,792 DEBUG [RS:2;f64cf8bdf19b:44077 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f64cf8bdf19b,44077,1732779879693' 2024-11-28T07:44:39,792 DEBUG [RS:2;f64cf8bdf19b:44077 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T07:44:39,792 DEBUG [RS:2;f64cf8bdf19b:44077 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T07:44:39,793 DEBUG [RS:2;f64cf8bdf19b:44077 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T07:44:39,793 INFO [RS:2;f64cf8bdf19b:44077 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T07:44:39,793 INFO [RS:2;f64cf8bdf19b:44077 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T07:44:39,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-28T07:44:39,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 375 msec 2024-11-28T07:44:39,794 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): f64cf8bdf19b,42741,1732779876434 had 2 regions 2024-11-28T07:44:39,794 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure f64cf8bdf19b,42741,1732779876434, splitWal=true, meta=true, isMeta: false 2024-11-28T07:44:39,795 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,42741,1732779876434-splitting dir is empty, no logs to split. 2024-11-28T07:44:39,795 INFO [PEWorker-3 {}] master.SplitWALManager(105): f64cf8bdf19b,42741,1732779876434 WAL count=0, meta=false 2024-11-28T07:44:39,796 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,42741,1732779876434-splitting dir is empty, no logs to split. 2024-11-28T07:44:39,796 INFO [PEWorker-3 {}] master.SplitWALManager(105): f64cf8bdf19b,42741,1732779876434 WAL count=0, meta=false 2024-11-28T07:44:39,796 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if f64cf8bdf19b,42741,1732779876434 WAL splitting is done? wals=0, meta=false 2024-11-28T07:44:39,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732779877496, region=4c22f09a39e6b82eb676be3e58313ff1, ASSIGN}] 2024-11-28T07:44:39,798 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732779877496, region=4c22f09a39e6b82eb676be3e58313ff1, ASSIGN 2024-11-28T07:44:39,799 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732779877496, region=4c22f09a39e6b82eb676be3e58313ff1, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-28T07:44:39,894 INFO [RS:2;f64cf8bdf19b:44077 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T07:44:39,895 INFO [RS:2;f64cf8bdf19b:44077 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f64cf8bdf19b%2C44077%2C1732779879693, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,44077,1732779879693, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/oldWALs, maxLogs=55 2024-11-28T07:44:39,916 INFO [RS:2;f64cf8bdf19b:44077 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/WALs/f64cf8bdf19b,44077,1732779879693/f64cf8bdf19b%2C44077%2C1732779879693.1732779879896 2024-11-28T07:44:39,916 DEBUG [RS:2;f64cf8bdf19b:44077 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T07:44:39,949 DEBUG [f64cf8bdf19b:43047 {}] balancer.BalancerClusterState(204): Hosts are {f64cf8bdf19b=0} racks are {/default-rack=0} 2024-11-28T07:44:39,949 DEBUG [f64cf8bdf19b:43047 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T07:44:39,950 DEBUG [f64cf8bdf19b:43047 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T07:44:39,950 DEBUG [f64cf8bdf19b:43047 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T07:44:39,950 DEBUG [f64cf8bdf19b:43047 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T07:44:39,950 INFO [f64cf8bdf19b:43047 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T07:44:39,950 INFO [f64cf8bdf19b:43047 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T07:44:39,950 DEBUG [f64cf8bdf19b:43047 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T07:44:39,950 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=4c22f09a39e6b82eb676be3e58313ff1, regionState=OPENING, regionLocation=f64cf8bdf19b,44077,1732779879693 2024-11-28T07:44:39,951 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f64cf8bdf19b:42741 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f64cf8bdf19b/172.17.0.2:42741 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:44:39,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,42741,1732779876434, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,42741,1732779876434, seqNum=-1, error=java.net.ConnectException: Call to address=f64cf8bdf19b:42741 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f64cf8bdf19b/172.17.0.2:42741 2024-11-28T07:44:39,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,42741,1732779876434, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-28T07:44:39,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,42741,1732779876434, seqNum=-1 from cache 2024-11-28T07:44:39,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address f64cf8bdf19b:42741 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f64cf8bdf19b/172.17.0.2:42741 2024-11-28T07:44:40,062 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:44:40,062 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,42865,1732779876461, seqNum=-1] 2024-11-28T07:44:40,063 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:44:40,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46547, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:44:40,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732779877496, region=4c22f09a39e6b82eb676be3e58313ff1, ASSIGN because future has completed 2024-11-28T07:44:40,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c22f09a39e6b82eb676be3e58313ff1, server=f64cf8bdf19b,44077,1732779879693}] 2024-11-28T07:44:40,220 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T07:44:40,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46617, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T07:44:40,224 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:40,224 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c22f09a39e6b82eb676be3e58313ff1, NAME => 'testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:44:40,225 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732779877496 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:40,225 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:44:40,225 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:40,225 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:40,226 INFO [StoreOpener-4c22f09a39e6b82eb676be3e58313ff1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:40,227 INFO [StoreOpener-4c22f09a39e6b82eb676be3e58313ff1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c22f09a39e6b82eb676be3e58313ff1 columnFamilyName family 2024-11-28T07:44:40,227 DEBUG [StoreOpener-4c22f09a39e6b82eb676be3e58313ff1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:44:40,229 DEBUG [StoreFileOpener-4c22f09a39e6b82eb676be3e58313ff1-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/family/cfebb5664b9f41178261084200076d26, delay=927 ms 2024-11-28T07:44:40,230 DEBUG [StoreOpener-4c22f09a39e6b82eb676be3e58313ff1-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/family/cfebb5664b9f41178261084200076d26 2024-11-28T07:44:40,230 INFO [StoreOpener-4c22f09a39e6b82eb676be3e58313ff1-1 {}] regionserver.HStore(327): Store=4c22f09a39e6b82eb676be3e58313ff1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:44:40,230 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:40,231 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:40,231 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:40,231 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:40,231 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:40,232 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:40,233 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 4c22f09a39e6b82eb676be3e58313ff1; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62810157, jitterRate=-0.06405572593212128}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T07:44:40,233 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:40,235 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 4c22f09a39e6b82eb676be3e58313ff1: Running coprocessor pre-open hook at 1732779880225Writing region info on filesystem at 1732779880225Initializing all the Stores at 1732779880226 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779880226Cleaning up temporary data from old regions at 1732779880231 (+5 ms)Running coprocessor post-open hooks at 1732779880233 (+2 ms)Region opened successfully at 1732779880234 (+1 ms) 2024-11-28T07:44:40,235 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1., pid=11, masterSystemTime=1732779880219 2024-11-28T07:44:40,238 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:40,238 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:40,238 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=4c22f09a39e6b82eb676be3e58313ff1, regionState=OPEN, openSeqNum=10, regionLocation=f64cf8bdf19b,44077,1732779879693 2024-11-28T07:44:40,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c22f09a39e6b82eb676be3e58313ff1, server=f64cf8bdf19b,44077,1732779879693 because future has completed 2024-11-28T07:44:40,244 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-28T07:44:40,244 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c22f09a39e6b82eb676be3e58313ff1, server=f64cf8bdf19b,44077,1732779879693 in 175 msec 2024-11-28T07:44:40,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-28T07:44:40,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732779877496, region=4c22f09a39e6b82eb676be3e58313ff1, ASSIGN in 447 msec 2024-11-28T07:44:40,246 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server f64cf8bdf19b,42741,1732779876434 after splitting done 2024-11-28T07:44:40,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure f64cf8bdf19b,42741,1732779876434, splitWal=true, meta=true in 841 msec 2024-11-28T07:44:40,290 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-28T07:44:40,290 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T07:44:40,291 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:44:40,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:40,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:40,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:44:40,291 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-28T07:44:40,291 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=116358662, stopped=false 2024-11-28T07:44:40,291 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f64cf8bdf19b,43047,1732779876381 2024-11-28T07:44:40,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T07:44:40,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T07:44:40,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44077-0x1015d888a9d0003, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T07:44:40,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:40,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:40,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44077-0x1015d888a9d0003, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:40,294 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:44:40,294 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T07:44:40,295 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:44:40,295 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:44:40,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:40,295 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44077-0x1015d888a9d0003, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:44:40,295 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f64cf8bdf19b,42865,1732779876461' ***** 2024-11-28T07:44:40,295 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T07:44:40,295 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:44:40,295 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f64cf8bdf19b,44077,1732779879693' ***** 2024-11-28T07:44:40,295 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T07:44:40,296 INFO [RS:1;f64cf8bdf19b:42865 {}] bucket.BucketCache(1738): Disabling cache 2024-11-28T07:44:40,296 INFO [RS:2;f64cf8bdf19b:44077 {}] bucket.BucketCache(1738): Disabling cache 2024-11-28T07:44:40,296 INFO [RS:1;f64cf8bdf19b:42865 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/bucket.persistence 2024-11-28T07:44:40,296 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-28T07:44:40,296 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-28T07:44:40,296 INFO [RS:2;f64cf8bdf19b:44077 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/89b514fe-b5d0-6e7e-4d33-97323be05376/bucket.persistence 2024-11-28T07:44:40,296 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-28T07:44:40,296 INFO [RS:1;f64cf8bdf19b:42865 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-28T07:44:40,296 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-28T07:44:40,296 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-28T07:44:40,296 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-28T07:44:40,296 INFO [RS:2;f64cf8bdf19b:44077 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-28T07:44:40,296 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-28T07:44:40,297 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-28T07:44:40,306 DEBUG [RS:1;f64cf8bdf19b:42865 {}] bucket.BucketCache(1487): Thread RS:1;f64cf8bdf19b:42865 started persisting bucket cache to file 2024-11-28T07:44:40,306 DEBUG [RS:1;f64cf8bdf19b:42865 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T07:44:40,306 DEBUG [RS:1;f64cf8bdf19b:42865 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T07:44:40,307 DEBUG [RS:2;f64cf8bdf19b:44077 {}] bucket.BucketCache(1487): Thread RS:2;f64cf8bdf19b:44077 started persisting bucket cache to file 2024-11-28T07:44:40,307 DEBUG [RS:2;f64cf8bdf19b:44077 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T07:44:40,307 DEBUG [RS:2;f64cf8bdf19b:44077 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T07:44:40,310 DEBUG [RS:1;f64cf8bdf19b:42865 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-28T07:44:40,310 DEBUG [RS:1;f64cf8bdf19b:42865 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732779880306 2024-11-28T07:44:40,310 DEBUG [RS:1;f64cf8bdf19b:42865 {}] bucket.BucketCache(1509): Thread RS:1;f64cf8bdf19b:42865 finished persisting bucket cache to file, renaming 2024-11-28T07:44:40,310 DEBUG [RS:2;f64cf8bdf19b:44077 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-28T07:44:40,310 DEBUG [RS:2;f64cf8bdf19b:44077 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732779880307 2024-11-28T07:44:40,310 DEBUG [RS:2;f64cf8bdf19b:44077 {}] bucket.BucketCache(1509): Thread RS:2;f64cf8bdf19b:44077 finished persisting bucket cache to file, renaming 2024-11-28T07:44:40,311 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T07:44:40,311 INFO [RS:1;f64cf8bdf19b:42865 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T07:44:40,311 INFO [RS:1;f64cf8bdf19b:42865 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T07:44:40,311 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T07:44:40,311 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.HRegionServer(959): stopping server f64cf8bdf19b,42865,1732779876461 2024-11-28T07:44:40,311 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T07:44:40,311 INFO [RS:1;f64cf8bdf19b:42865 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f64cf8bdf19b:42865. 2024-11-28T07:44:40,311 DEBUG [RS:1;f64cf8bdf19b:42865 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:44:40,311 DEBUG [RS:1;f64cf8bdf19b:42865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:40,311 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T07:44:40,311 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T07:44:40,311 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T07:44:40,311 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T07:44:40,311 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T07:44:40,311 INFO [RS:2;f64cf8bdf19b:44077 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T07:44:40,311 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T07:44:40,311 INFO [RS:2;f64cf8bdf19b:44077 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T07:44:40,311 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.HRegionServer(3091): Received CLOSE for 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:40,312 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-28T07:44:40,312 DEBUG [RS:1;f64cf8bdf19b:42865 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-28T07:44:40,312 DEBUG [RS:1;f64cf8bdf19b:42865 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T07:44:40,312 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.HRegionServer(959): stopping server f64cf8bdf19b,44077,1732779879693 2024-11-28T07:44:40,312 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T07:44:40,312 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T07:44:40,312 INFO [RS:2;f64cf8bdf19b:44077 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;f64cf8bdf19b:44077. 2024-11-28T07:44:40,312 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T07:44:40,312 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T07:44:40,312 DEBUG [RS:2;f64cf8bdf19b:44077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:44:40,312 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4c22f09a39e6b82eb676be3e58313ff1, disabling compactions & flushes 2024-11-28T07:44:40,312 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T07:44:40,312 DEBUG [RS:2;f64cf8bdf19b:44077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:40,312 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:40,312 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T07:44:40,312 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-28T07:44:40,312 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T07:44:40,312 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:40,312 DEBUG [RS:2;f64cf8bdf19b:44077 {}] regionserver.HRegionServer(1325): Online Regions={4c22f09a39e6b82eb676be3e58313ff1=testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1.} 2024-11-28T07:44:40,312 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T07:44:40,312 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. after waiting 0 ms 2024-11-28T07:44:40,312 DEBUG [RS:2;f64cf8bdf19b:44077 {}] regionserver.HRegionServer(1351): Waiting on 4c22f09a39e6b82eb676be3e58313ff1 2024-11-28T07:44:40,312 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:40,313 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-28T07:44:40,313 DEBUG [StoreFileCloser-4c22f09a39e6b82eb676be3e58313ff1-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/family/cfebb5664b9f41178261084200076d26 2024-11-28T07:44:40,318 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/default/testBlockEvictionOnGracefulStop1732779877496/4c22f09a39e6b82eb676be3e58313ff1/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-28T07:44:40,318 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:40,318 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4c22f09a39e6b82eb676be3e58313ff1: Waiting for close lock at 1732779880312Running coprocessor pre-close hooks at 1732779880312Disabling compacts and flushes for region at 1732779880312Disabling writes for close at 1732779880312Writing region close event to WAL at 1732779880313 (+1 ms)Running coprocessor post-close hooks at 1732779880318 (+5 ms)Closed at 1732779880318 2024-11-28T07:44:40,319 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1. 2024-11-28T07:44:40,323 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4d4f0c3b12bb4affa6d917084bf4cf71 2024-11-28T07:44:40,338 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/info/4d4f0c3b12bb4affa6d917084bf4cf71 is 209, key is testBlockEvictionOnGracefulStop1732779877496,,1732779877499.4c22f09a39e6b82eb676be3e58313ff1./info:regioninfo/1732779880238/Put/seqid=0 2024-11-28T07:44:40,339 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/info/4d4f0c3b12bb4affa6d917084bf4cf71 2024-11-28T07:44:40,342 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/.tmp/info/4d4f0c3b12bb4affa6d917084bf4cf71 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/info/4d4f0c3b12bb4affa6d917084bf4cf71 2024-11-28T07:44:40,342 INFO [regionserver/f64cf8bdf19b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:44:40,343 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/info/4d4f0c3b12bb4affa6d917084bf4cf71, delay=909 ms 2024-11-28T07:44:40,344 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/info/4d4f0c3b12bb4affa6d917084bf4cf71, entries=8, sequenceid=19, filesize=6.6 K 2024-11-28T07:44:40,345 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 33ms, sequenceid=19, compaction requested=false 2024-11-28T07:44:40,346 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/info/4bf69875c4864968b4c68b0d76a94382 2024-11-28T07:44:40,346 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/info/4d4f0c3b12bb4affa6d917084bf4cf71 2024-11-28T07:44:40,346 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/ns/c523d09e42e64248aa9202a094f5287a 2024-11-28T07:44:40,348 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/table/2d417ac8343544ef8ae37b4aca43ddbd 2024-11-28T07:44:40,353 INFO [regionserver/f64cf8bdf19b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:44:40,353 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-28T07:44:40,353 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T07:44:40,353 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T07:44:40,354 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732779880312Running coprocessor pre-close hooks at 1732779880312Disabling compacts and flushes for region at 1732779880312Disabling writes for close at 1732779880312Obtaining lock to block concurrent updates at 1732779880313 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732779880313Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732779880313Flushing stores of hbase:meta,,1.1588230740 at 1732779880313Flushing 1588230740/info: creating writer at 1732779880313Flushing 1588230740/info: appending metadata at 1732779880337 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732779880337Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47ee90c3: reopening flushed file at 1732779880341 (+4 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 33ms, sequenceid=19, compaction requested=false at 1732779880345 (+4 ms)Writing region close event to WAL at 1732779880348 (+3 ms)Running coprocessor post-close hooks at 1732779880353 (+5 ms)Closed at 1732779880353 2024-11-28T07:44:40,354 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T07:44:40,512 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.HRegionServer(976): stopping server f64cf8bdf19b,42865,1732779876461; all regions closed. 2024-11-28T07:44:40,513 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.HRegionServer(976): stopping server f64cf8bdf19b,44077,1732779879693; all regions closed. 2024-11-28T07:44:40,514 DEBUG [RS:1;f64cf8bdf19b:42865 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/oldWALs 2024-11-28T07:44:40,514 INFO [RS:1;f64cf8bdf19b:42865 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f64cf8bdf19b%2C42865%2C1732779876461.meta:.meta(num 1732779879730) 2024-11-28T07:44:40,514 DEBUG [RS:2;f64cf8bdf19b:44077 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/oldWALs 2024-11-28T07:44:40,514 INFO [RS:2;f64cf8bdf19b:44077 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f64cf8bdf19b%2C44077%2C1732779879693:(num 1732779879896) 2024-11-28T07:44:40,514 DEBUG [RS:2;f64cf8bdf19b:44077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:40,515 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:44:40,515 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T07:44:40,515 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.ChoreService(370): Chore service for: regionserver/f64cf8bdf19b:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-28T07:44:40,515 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T07:44:40,515 INFO [regionserver/f64cf8bdf19b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T07:44:40,515 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T07:44:40,515 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T07:44:40,515 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T07:44:40,515 INFO [RS:2;f64cf8bdf19b:44077 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44077 2024-11-28T07:44:40,516 DEBUG [RS:1;f64cf8bdf19b:42865 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/oldWALs 2024-11-28T07:44:40,516 INFO [RS:1;f64cf8bdf19b:42865 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f64cf8bdf19b%2C42865%2C1732779876461:(num 1732779876986) 2024-11-28T07:44:40,516 DEBUG [RS:1;f64cf8bdf19b:42865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:44:40,516 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:44:40,517 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T07:44:40,517 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.ChoreService(370): Chore service for: regionserver/f64cf8bdf19b:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-28T07:44:40,517 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T07:44:40,517 INFO [regionserver/f64cf8bdf19b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T07:44:40,517 INFO [RS:1;f64cf8bdf19b:42865 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42865 2024-11-28T07:44:40,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44077-0x1015d888a9d0003, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f64cf8bdf19b,44077,1732779879693 2024-11-28T07:44:40,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T07:44:40,518 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T07:44:40,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f64cf8bdf19b,42865,1732779876461 2024-11-28T07:44:40,520 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f64cf8bdf19b,42865,1732779876461] 2024-11-28T07:44:40,520 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T07:44:40,522 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f64cf8bdf19b,42865,1732779876461 already deleted, retry=false 2024-11-28T07:44:40,522 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f64cf8bdf19b,42865,1732779876461 expired; onlineServers=1 2024-11-28T07:44:40,522 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f64cf8bdf19b,44077,1732779879693] 2024-11-28T07:44:40,523 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f64cf8bdf19b,44077,1732779879693 already deleted, retry=false 2024-11-28T07:44:40,523 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f64cf8bdf19b,44077,1732779879693 expired; onlineServers=0 2024-11-28T07:44:40,523 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f64cf8bdf19b,43047,1732779876381' ***** 2024-11-28T07:44:40,523 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-28T07:44:40,523 INFO [M:0;f64cf8bdf19b:43047 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T07:44:40,523 INFO [M:0;f64cf8bdf19b:43047 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T07:44:40,523 DEBUG [M:0;f64cf8bdf19b:43047 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-28T07:44:40,523 DEBUG [M:0;f64cf8bdf19b:43047 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-28T07:44:40,523 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-28T07:44:40,523 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.large.0-1732779876711 {}] cleaner.HFileCleaner(306): Exit Thread[master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.large.0-1732779876711,5,FailOnTimeoutGroup] 2024-11-28T07:44:40,523 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.small.0-1732779876712 {}] cleaner.HFileCleaner(306): Exit Thread[master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.small.0-1732779876712,5,FailOnTimeoutGroup] 2024-11-28T07:44:40,524 INFO [M:0;f64cf8bdf19b:43047 {}] hbase.ChoreService(370): Chore service for: master/f64cf8bdf19b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-28T07:44:40,524 INFO [M:0;f64cf8bdf19b:43047 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T07:44:40,524 DEBUG [M:0;f64cf8bdf19b:43047 {}] master.HMaster(1795): Stopping service threads 2024-11-28T07:44:40,524 INFO [M:0;f64cf8bdf19b:43047 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-28T07:44:40,524 INFO [M:0;f64cf8bdf19b:43047 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:44:40,524 INFO [M:0;f64cf8bdf19b:43047 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:44:40,524 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-28T07:44:40,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-28T07:44:40,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:44:40,526 DEBUG [M:0;f64cf8bdf19b:43047 {}] zookeeper.ZKUtil(347): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-28T07:44:40,526 WARN [M:0;f64cf8bdf19b:43047 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-28T07:44:40,526 INFO [M:0;f64cf8bdf19b:43047 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/.lastflushedseqids 2024-11-28T07:44:40,530 INFO [M:0;f64cf8bdf19b:43047 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:44:40,530 INFO [M:0;f64cf8bdf19b:43047 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-28T07:44:40,530 DEBUG [M:0;f64cf8bdf19b:43047 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T07:44:40,530 INFO [M:0;f64cf8bdf19b:43047 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:44:40,530 DEBUG [M:0;f64cf8bdf19b:43047 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:44:40,530 DEBUG [M:0;f64cf8bdf19b:43047 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T07:44:40,530 DEBUG [M:0;f64cf8bdf19b:43047 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:44:40,530 INFO [M:0;f64cf8bdf19b:43047 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.51 KB heapSize=49.91 KB 2024-11-28T07:44:40,542 DEBUG [M:0;f64cf8bdf19b:43047 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a0ee8b38a4294f6994844643a36ab3d7 2024-11-28T07:44:40,557 DEBUG [M:0;f64cf8bdf19b:43047 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a0ee8b38a4294f6994844643a36ab3d7 is 82, key is hbase:meta,,1/info:regioninfo/1732779879786/Put/seqid=0 2024-11-28T07:44:40,557 INFO [M:0;f64cf8bdf19b:43047 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a0ee8b38a4294f6994844643a36ab3d7 2024-11-28T07:44:40,566 DEBUG [M:0;f64cf8bdf19b:43047 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cd842dde7bb14255adac2979a6d51798 2024-11-28T07:44:40,580 DEBUG [M:0;f64cf8bdf19b:43047 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cd842dde7bb14255adac2979a6d51798 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732779877869/Put/seqid=0 2024-11-28T07:44:40,580 INFO [M:0;f64cf8bdf19b:43047 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.28 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cd842dde7bb14255adac2979a6d51798 2024-11-28T07:44:40,589 DEBUG [M:0;f64cf8bdf19b:43047 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2b3e742c95054f338c3c2960623b361b 2024-11-28T07:44:40,610 DEBUG [M:0;f64cf8bdf19b:43047 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2b3e742c95054f338c3c2960623b361b is 69, key is f64cf8bdf19b,42865,1732779876461/rs:state/1732779876809/Put/seqid=0 2024-11-28T07:44:40,611 INFO [M:0;f64cf8bdf19b:43047 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2b3e742c95054f338c3c2960623b361b 2024-11-28T07:44:40,614 INFO [M:0;f64cf8bdf19b:43047 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2b3e742c95054f338c3c2960623b361b 2024-11-28T07:44:40,614 DEBUG [M:0;f64cf8bdf19b:43047 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a0ee8b38a4294f6994844643a36ab3d7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a0ee8b38a4294f6994844643a36ab3d7 2024-11-28T07:44:40,617 INFO [M:0;f64cf8bdf19b:43047 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a0ee8b38a4294f6994844643a36ab3d7, entries=14, sequenceid=102, filesize=5.9 K 2024-11-28T07:44:40,618 DEBUG [M:0;f64cf8bdf19b:43047 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cd842dde7bb14255adac2979a6d51798 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cd842dde7bb14255adac2979a6d51798 2024-11-28T07:44:40,620 INFO [M:0;f64cf8bdf19b:43047 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cd842dde7bb14255adac2979a6d51798, entries=11, sequenceid=102, filesize=6.5 K 2024-11-28T07:44:40,620 DEBUG [M:0;f64cf8bdf19b:43047 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2b3e742c95054f338c3c2960623b361b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2b3e742c95054f338c3c2960623b361b 2024-11-28T07:44:40,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44077-0x1015d888a9d0003, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:44:40,621 INFO [RS:2;f64cf8bdf19b:44077 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T07:44:40,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44077-0x1015d888a9d0003, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:44:40,621 INFO [RS:2;f64cf8bdf19b:44077 {}] regionserver.HRegionServer(1031): Exiting; stopping=f64cf8bdf19b,44077,1732779879693; zookeeper connection closed. 2024-11-28T07:44:40,622 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@368c9fb5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@368c9fb5 2024-11-28T07:44:40,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:44:40,622 INFO [RS:1;f64cf8bdf19b:42865 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T07:44:40,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42865-0x1015d888a9d0002, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:44:40,622 INFO [RS:1;f64cf8bdf19b:42865 {}] regionserver.HRegionServer(1031): Exiting; stopping=f64cf8bdf19b,42865,1732779876461; zookeeper connection closed. 2024-11-28T07:44:40,622 INFO [M:0;f64cf8bdf19b:43047 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2b3e742c95054f338c3c2960623b361b 2024-11-28T07:44:40,622 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@180d6b4a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@180d6b4a 2024-11-28T07:44:40,622 INFO [M:0;f64cf8bdf19b:43047 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3d6c1e2a-0f59-71b2-688c-4ca075f1ff8e/dd751227-0368-ffce-d7ab-03e310a62597/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2b3e742c95054f338c3c2960623b361b, entries=3, sequenceid=102, filesize=5.3 K 2024-11-28T07:44:40,623 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-28T07:44:40,623 INFO [M:0;f64cf8bdf19b:43047 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 93ms, sequenceid=102, compaction requested=false 2024-11-28T07:44:40,626 INFO [M:0;f64cf8bdf19b:43047 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:44:40,626 DEBUG [M:0;f64cf8bdf19b:43047 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732779880530Disabling compacts and flushes for region at 1732779880530Disabling writes for close at 1732779880530Obtaining lock to block concurrent updates at 1732779880530Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732779880530Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40460, getHeapSize=51048, getOffHeapSize=0, getCellsCount=127 at 1732779880531 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732779880531Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732779880531Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732779880556 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732779880556Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732779880559 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732779880579 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732779880579Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732779880582 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732779880609 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732779880609Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@192ec437: reopening flushed file at 1732779880614 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d79a4b9: reopening flushed file at 1732779880617 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c8af806: reopening flushed file at 1732779880620 (+3 ms)Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 93ms, sequenceid=102, compaction requested=false at 1732779880623 (+3 ms)Writing region close event to WAL at 1732779880625 (+2 ms)Closed at 1732779880625 2024-11-28T07:44:40,627 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T07:44:40,627 INFO [M:0;f64cf8bdf19b:43047 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-28T07:44:40,627 INFO [M:0;f64cf8bdf19b:43047 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43047 2024-11-28T07:44:40,627 INFO [M:0;f64cf8bdf19b:43047 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T07:44:40,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:44:40,730 INFO [M:0;f64cf8bdf19b:43047 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T07:44:40,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43047-0x1015d888a9d0000, quorum=127.0.0.1:64536, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:44:40,733 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-28T07:44:40,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-28T07:44:40,734 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:44:40,742 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 50) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=312 (was 339), ProcessCount=11 (was 11), AvailableMemoryMB=6968 (was 7105)