2024-11-25 03:45:26,671 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-25 03:45:26,684 main DEBUG Took 0.009658 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 03:45:26,684 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 03:45:26,684 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 03:45:26,692 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 03:45:26,694 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:45:26,704 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 03:45:26,719 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:45:26,720 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:45:26,721 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:45:26,721 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:45:26,722 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:45:26,722 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:45:26,723 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:45:26,723 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:45:26,724 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:45:26,724 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:45:26,725 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:45:26,726 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:45:26,727 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:45:26,727 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:45:26,727 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:45:26,728 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:45:26,728 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:45:26,729 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:45:26,729 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:45:26,730 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:45:26,730 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:45:26,730 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:45:26,731 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:45:26,731 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:45:26,732 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:45:26,732 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 03:45:26,734 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:45:26,736 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 03:45:26,738 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 03:45:26,738 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 03:45:26,740 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 03:45:26,741 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 03:45:26,751 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 03:45:26,754 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 03:45:26,756 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 03:45:26,756 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 03:45:26,756 main DEBUG createAppenders(={Console}) 2024-11-25 03:45:26,757 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-25 03:45:26,758 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-25 03:45:26,758 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-25 03:45:26,759 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 03:45:26,759 main DEBUG OutputStream closed 2024-11-25 03:45:26,759 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 03:45:26,760 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 03:45:26,760 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-25 03:45:26,857 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 03:45:26,860 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 03:45:26,861 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 03:45:26,862 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 03:45:26,862 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 03:45:26,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 03:45:26,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 03:45:26,864 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 03:45:26,864 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 03:45:26,865 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 03:45:26,865 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 03:45:26,866 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 03:45:26,866 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 03:45:26,867 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 03:45:26,867 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 03:45:26,867 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 03:45:26,867 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 03:45:26,868 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 03:45:26,870 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 03:45:26,871 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-25 03:45:26,871 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 03:45:26,872 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-25T03:45:26,890 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-25 03:45:26,894 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 03:45:26,894 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T03:45:27,157 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f 2024-11-25T03:45:27,180 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=398, ProcessCount=11, AvailableMemoryMB=9678 2024-11-25T03:45:27,267 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T03:45:27,355 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:27,370 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/cluster_b5e96a76-15de-53d1-01e8-432b4cd766a4, deleteOnExit=true 2024-11-25T03:45:27,457 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/cluster_b5e96a76-15de-53d1-01e8-432b4cd766a4/zookeeper_0, clientPort=50232, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/cluster_b5e96a76-15de-53d1-01e8-432b4cd766a4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/cluster_b5e96a76-15de-53d1-01e8-432b4cd766a4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T03:45:27,467 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50232 2024-11-25T03:45:27,468 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:27,545 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1 with version=8 2024-11-25T03:45:27,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/hbase-staging 2024-11-25T03:45:27,635 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T03:45:27,890 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d73269e94d72:0 server-side Connection retries=45 2024-11-25T03:45:27,899 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:27,901 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:27,911 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:45:27,911 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:27,912 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:45:28,044 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T03:45:28,127 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:45:28,170 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T03:45:28,180 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T03:45:28,184 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:45:28,229 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 15025 (auto-detected) 2024-11-25T03:45:28,230 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-25T03:45:28,253 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45215 2024-11-25T03:45:28,281 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45215 connecting to ZooKeeper ensemble=127.0.0.1:50232 2024-11-25T03:45:28,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:452150x0, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:45:28,540 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45215-0x101701895360000 connected 2024-11-25T03:45:29,034 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:29,036 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:29,043 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:45:29,046 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1, hbase.cluster.distributed=false 2024-11-25T03:45:29,068 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:45:29,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45215 2024-11-25T03:45:29,074 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45215 2024-11-25T03:45:29,074 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45215 2024-11-25T03:45:29,075 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45215 2024-11-25T03:45:29,075 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45215 2024-11-25T03:45:29,152 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d73269e94d72:0 server-side Connection retries=45 2024-11-25T03:45:29,153 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:29,154 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:29,154 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:45:29,154 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:29,154 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:45:29,157 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T03:45:29,160 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:45:29,161 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39133 2024-11-25T03:45:29,163 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39133 connecting to ZooKeeper ensemble=127.0.0.1:50232 2024-11-25T03:45:29,164 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:29,165 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:29,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:391330x0, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:45:29,251 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:45:29,251 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39133-0x101701895360001 connected 2024-11-25T03:45:29,261 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T03:45:29,278 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/bucket.cache 2024-11-25T03:45:29,285 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T03:45:29,287 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T03:45:29,289 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T03:45:29,290 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T03:45:29,291 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-25T03:45:29,297 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T03:45:29,303 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T03:45:29,303 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T03:45:29,303 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/bucket.persistence is complete. 2024-11-25T03:45:29,306 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T03:45:29,313 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:45:29,314 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39133 2024-11-25T03:45:29,317 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39133 2024-11-25T03:45:29,319 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39133 2024-11-25T03:45:29,320 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39133 2024-11-25T03:45:29,322 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39133 2024-11-25T03:45:29,342 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d73269e94d72:0 server-side Connection retries=45 2024-11-25T03:45:29,342 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:29,343 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:29,343 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:45:29,343 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:29,343 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:45:29,344 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T03:45:29,344 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:45:29,345 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42927 2024-11-25T03:45:29,347 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42927 connecting to ZooKeeper ensemble=127.0.0.1:50232 2024-11-25T03:45:29,348 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:29,349 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:29,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:429270x0, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:45:29,403 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42927-0x101701895360002 connected 2024-11-25T03:45:29,403 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:429270x0, quorum=127.0.0.1:50232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:45:29,404 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T03:45:29,405 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/bucket.cache 2024-11-25T03:45:29,406 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T03:45:29,406 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T03:45:29,406 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T03:45:29,407 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T03:45:29,407 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T03:45:29,407 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-25T03:45:29,408 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T03:45:29,408 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/bucket.persistence is complete. 2024-11-25T03:45:29,408 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T03:45:29,409 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T03:45:29,412 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:45:29,415 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42927 2024-11-25T03:45:29,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42927 2024-11-25T03:45:29,418 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42927 2024-11-25T03:45:29,420 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42927 2024-11-25T03:45:29,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42927 2024-11-25T03:45:29,441 DEBUG [M:0;d73269e94d72:45215 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d73269e94d72:45215 2024-11-25T03:45:29,444 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d73269e94d72,45215,1732506327714 2024-11-25T03:45:29,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:45:29,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:45:29,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:45:29,543 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d73269e94d72,45215,1732506327714 2024-11-25T03:45:29,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:29,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T03:45:29,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T03:45:29,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:29,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:29,608 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T03:45:29,609 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d73269e94d72,45215,1732506327714 from backup master directory 2024-11-25T03:45:29,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:45:29,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:45:29,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d73269e94d72,45215,1732506327714 2024-11-25T03:45:29,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:45:29,674 WARN [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:45:29,674 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d73269e94d72,45215,1732506327714 2024-11-25T03:45:29,676 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T03:45:29,678 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T03:45:29,686 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/hbase.id] with ID: 56a01f60-e64f-4f65-a758-996fad6bc065 2024-11-25T03:45:29,686 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/.tmp/hbase.id 2024-11-25T03:45:29,692 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/hbase.id] 2024-11-25T03:45:29,726 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:29,733 INFO [master/d73269e94d72:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T03:45:29,756 INFO [master/d73269e94d72:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-25T03:45:29,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:29,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:29,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:29,844 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:45:29,847 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T03:45:29,885 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:45:29,890 INFO [master/d73269e94d72:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:45:29,958 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store 2024-11-25T03:45:29,981 INFO [master/d73269e94d72:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T03:45:29,986 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:45:29,988 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T03:45:29,988 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:45:29,989 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:45:29,991 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T03:45:29,991 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:45:29,992 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:45:29,993 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732506329988Disabling compacts and flushes for region at 1732506329988Disabling writes for close at 1732506329991 (+3 ms)Writing region close event to WAL at 1732506329992 (+1 ms)Closed at 1732506329992 2024-11-25T03:45:29,993 WARN [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/.initializing 2024-11-25T03:45:29,994 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/WALs/d73269e94d72,45215,1732506327714 2024-11-25T03:45:30,004 INFO [master/d73269e94d72:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:45:30,015 INFO [master/d73269e94d72:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C45215%2C1732506327714, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/WALs/d73269e94d72,45215,1732506327714, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/oldWALs, maxLogs=10 2024-11-25T03:45:30,049 INFO [master/d73269e94d72:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/WALs/d73269e94d72,45215,1732506327714/d73269e94d72%2C45215%2C1732506327714.1732506330019 2024-11-25T03:45:30,050 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T03:45:30,051 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:45:30,051 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:45:30,055 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:30,056 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:30,092 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:30,122 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T03:45:30,126 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:30,129 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:30,129 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:30,136 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T03:45:30,136 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:30,136 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:45:30,137 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:30,142 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T03:45:30,142 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:30,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:45:30,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:30,149 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T03:45:30,149 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:30,150 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:45:30,150 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:30,155 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:30,156 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:30,160 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:30,161 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:30,165 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:45:30,168 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:30,177 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:45:30,178 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72466667, jitterRate=0.07983748614788055}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:45:30,186 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732506330071Initializing all the Stores at 1732506330071Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506330072 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506330073 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506330073Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506330073Cleaning up temporary data from old regions at 1732506330161 (+88 ms)Region opened successfully at 1732506330186 (+25 ms) 2024-11-25T03:45:30,188 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T03:45:30,219 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f8d8b2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:45:30,252 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T03:45:30,265 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T03:45:30,265 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T03:45:30,269 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:45:30,270 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:45:30,279 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-25T03:45:30,279 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T03:45:30,309 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T03:45:30,319 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T03:45:30,341 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T03:45:30,344 INFO [master/d73269e94d72:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T03:45:30,346 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T03:45:30,372 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T03:45:30,375 INFO [master/d73269e94d72:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T03:45:30,379 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T03:45:30,456 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T03:45:30,460 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T03:45:30,488 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T03:45:30,512 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T03:45:30,530 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T03:45:30,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:45:30,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:45:30,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:45:30,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:30,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:30,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:30,568 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d73269e94d72,45215,1732506327714, sessionid=0x101701895360000, setting cluster-up flag (Was=false) 2024-11-25T03:45:30,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:30,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:30,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:30,635 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T03:45:30,638 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d73269e94d72,45215,1732506327714 2024-11-25T03:45:30,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:30,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:30,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:30,709 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T03:45:30,713 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d73269e94d72,45215,1732506327714 2024-11-25T03:45:30,720 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T03:45:30,729 INFO [RS:0;d73269e94d72:39133 {}] regionserver.HRegionServer(746): ClusterId : 56a01f60-e64f-4f65-a758-996fad6bc065 2024-11-25T03:45:30,729 INFO [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(746): ClusterId : 56a01f60-e64f-4f65-a758-996fad6bc065 2024-11-25T03:45:30,732 DEBUG [RS:0;d73269e94d72:39133 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T03:45:30,732 DEBUG [RS:1;d73269e94d72:42927 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T03:45:30,753 DEBUG [RS:0;d73269e94d72:39133 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T03:45:30,753 DEBUG [RS:1;d73269e94d72:42927 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T03:45:30,754 DEBUG [RS:0;d73269e94d72:39133 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T03:45:30,754 DEBUG [RS:1;d73269e94d72:42927 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T03:45:30,763 DEBUG [RS:1;d73269e94d72:42927 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T03:45:30,763 DEBUG [RS:0;d73269e94d72:39133 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T03:45:30,764 DEBUG [RS:0;d73269e94d72:39133 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ec76c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:45:30,764 DEBUG [RS:1;d73269e94d72:42927 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c1868cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:45:30,781 DEBUG [RS:0;d73269e94d72:39133 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d73269e94d72:39133 2024-11-25T03:45:30,785 DEBUG [RS:1;d73269e94d72:42927 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;d73269e94d72:42927 2024-11-25T03:45:30,786 INFO [RS:0;d73269e94d72:39133 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T03:45:30,786 INFO [RS:1;d73269e94d72:42927 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T03:45:30,786 INFO [RS:0;d73269e94d72:39133 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T03:45:30,786 INFO [RS:1;d73269e94d72:42927 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T03:45:30,786 DEBUG [RS:0;d73269e94d72:39133 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T03:45:30,786 DEBUG [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T03:45:30,789 INFO [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,45215,1732506327714 with port=42927, startcode=1732506329341 2024-11-25T03:45:30,789 INFO [RS:0;d73269e94d72:39133 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,45215,1732506327714 with port=39133, startcode=1732506329119 2024-11-25T03:45:30,804 DEBUG [RS:0;d73269e94d72:39133 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T03:45:30,804 DEBUG [RS:1;d73269e94d72:42927 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T03:45:30,803 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:45:30,818 INFO [master/d73269e94d72:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T03:45:30,827 INFO [master/d73269e94d72:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T03:45:30,832 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d73269e94d72,45215,1732506327714 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T03:45:30,840 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:45:30,841 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:45:30,841 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:45:30,841 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:45:30,841 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d73269e94d72:0, corePoolSize=10, maxPoolSize=10 2024-11-25T03:45:30,841 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:30,842 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:45:30,842 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:30,848 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:45:30,851 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T03:45:30,857 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:30,857 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T03:45:30,861 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506360861 2024-11-25T03:45:30,863 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T03:45:30,864 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T03:45:30,868 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T03:45:30,868 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T03:45:30,868 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T03:45:30,869 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T03:45:30,882 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48557, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T03:45:30,882 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53599, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T03:45:30,876 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:30,885 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T03:45:30,886 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T03:45:30,887 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T03:45:30,890 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T03:45:30,891 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T03:45:30,892 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45215 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:45:30,896 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.large.0-1732506330892,5,FailOnTimeoutGroup] 2024-11-25T03:45:30,897 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T03:45:30,897 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.small.0-1732506330897,5,FailOnTimeoutGroup] 2024-11-25T03:45:30,897 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:30,898 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T03:45:30,898 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1 2024-11-25T03:45:30,898 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45215 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T03:45:30,899 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:30,899 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:30,920 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:45:30,921 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T03:45:30,924 DEBUG [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-25T03:45:30,924 DEBUG [RS:0;d73269e94d72:39133 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-25T03:45:30,924 WARN [RS:0;d73269e94d72:39133 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-25T03:45:30,924 WARN [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-25T03:45:30,930 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:45:30,930 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:30,930 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:30,931 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T03:45:30,937 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:45:30,937 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:30,937 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:30,938 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T03:45:30,941 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:45:30,941 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:30,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:30,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T03:45:30,945 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:45:30,946 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:30,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:30,946 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:45:30,947 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740 2024-11-25T03:45:30,947 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740 2024-11-25T03:45:30,948 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:45:30,948 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:45:30,948 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:45:30,949 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:45:30,955 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:45:30,956 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62354746, jitterRate=-0.07084187865257263}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:45:30,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732506330921Initializing all the Stores at 1732506330921Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506330921Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506330921Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506330921Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506330921Cleaning up temporary data from old regions at 1732506330948 (+27 ms)Region opened successfully at 1732506330960 (+12 ms) 2024-11-25T03:45:30,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:45:30,960 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:45:30,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:45:30,961 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:45:30,961 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:45:30,963 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:45:30,963 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732506330960Disabling compacts and flushes for region at 1732506330960Disabling writes for close at 1732506330961 (+1 ms)Writing region close event to WAL at 1732506330962 (+1 ms)Closed at 1732506330962 2024-11-25T03:45:30,966 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:45:30,966 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T03:45:30,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T03:45:30,982 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:45:30,984 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T03:45:31,026 INFO [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,45215,1732506327714 with port=42927, startcode=1732506329341 2024-11-25T03:45:31,026 INFO [RS:0;d73269e94d72:39133 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,45215,1732506327714 with port=39133, startcode=1732506329119 2024-11-25T03:45:31,028 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45215 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d73269e94d72,42927,1732506329341 2024-11-25T03:45:31,030 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45215 {}] master.ServerManager(517): Registering regionserver=d73269e94d72,42927,1732506329341 2024-11-25T03:45:31,036 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45215 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d73269e94d72,39133,1732506329119 2024-11-25T03:45:31,036 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45215 {}] master.ServerManager(517): Registering regionserver=d73269e94d72,39133,1732506329119 2024-11-25T03:45:31,036 DEBUG [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1 2024-11-25T03:45:31,036 DEBUG [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T03:45:31,036 DEBUG [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T03:45:31,038 DEBUG [RS:0;d73269e94d72:39133 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1 2024-11-25T03:45:31,038 DEBUG [RS:0;d73269e94d72:39133 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T03:45:31,038 DEBUG [RS:0;d73269e94d72:39133 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T03:45:31,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:45:31,111 DEBUG [RS:1;d73269e94d72:42927 {}] zookeeper.ZKUtil(111): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d73269e94d72,42927,1732506329341 2024-11-25T03:45:31,111 DEBUG [RS:0;d73269e94d72:39133 {}] zookeeper.ZKUtil(111): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d73269e94d72,39133,1732506329119 2024-11-25T03:45:31,111 WARN [RS:1;d73269e94d72:42927 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:45:31,111 WARN [RS:0;d73269e94d72:39133 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:45:31,112 INFO [RS:1;d73269e94d72:42927 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:45:31,112 INFO [RS:0;d73269e94d72:39133 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:45:31,112 DEBUG [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,42927,1732506329341 2024-11-25T03:45:31,112 DEBUG [RS:0;d73269e94d72:39133 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,39133,1732506329119 2024-11-25T03:45:31,115 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d73269e94d72,39133,1732506329119] 2024-11-25T03:45:31,116 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d73269e94d72,42927,1732506329341] 2024-11-25T03:45:31,135 WARN [d73269e94d72:45215 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:45:31,138 INFO [RS:1;d73269e94d72:42927 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T03:45:31,138 INFO [RS:0;d73269e94d72:39133 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T03:45:31,155 INFO [RS:0;d73269e94d72:39133 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T03:45:31,155 INFO [RS:1;d73269e94d72:42927 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T03:45:31,160 INFO [RS:0;d73269e94d72:39133 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:45:31,160 INFO [RS:1;d73269e94d72:42927 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:45:31,160 INFO [RS:0;d73269e94d72:39133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,160 INFO [RS:1;d73269e94d72:42927 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,161 INFO [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T03:45:31,161 INFO [RS:0;d73269e94d72:39133 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T03:45:31,165 INFO [RS:0;d73269e94d72:39133 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T03:45:31,165 INFO [RS:1;d73269e94d72:42927 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T03:45:31,167 INFO [RS:0;d73269e94d72:39133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,167 INFO [RS:1;d73269e94d72:42927 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,167 DEBUG [RS:0;d73269e94d72:39133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,167 DEBUG [RS:1;d73269e94d72:42927 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,167 DEBUG [RS:0;d73269e94d72:39133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,167 DEBUG [RS:1;d73269e94d72:42927 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,167 DEBUG [RS:0;d73269e94d72:39133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,167 DEBUG [RS:1;d73269e94d72:42927 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,168 DEBUG [RS:0;d73269e94d72:39133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,168 DEBUG [RS:0;d73269e94d72:39133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,168 DEBUG [RS:1;d73269e94d72:42927 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,168 DEBUG [RS:0;d73269e94d72:39133 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:45:31,168 DEBUG [RS:1;d73269e94d72:42927 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,168 DEBUG [RS:0;d73269e94d72:39133 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,168 DEBUG [RS:1;d73269e94d72:42927 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:45:31,168 DEBUG [RS:0;d73269e94d72:39133 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,168 DEBUG [RS:0;d73269e94d72:39133 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,168 DEBUG [RS:1;d73269e94d72:42927 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,168 DEBUG [RS:0;d73269e94d72:39133 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,168 DEBUG [RS:1;d73269e94d72:42927 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,169 DEBUG [RS:0;d73269e94d72:39133 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,169 DEBUG [RS:0;d73269e94d72:39133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,169 DEBUG [RS:1;d73269e94d72:42927 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,169 DEBUG [RS:0;d73269e94d72:39133 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:45:31,169 DEBUG [RS:1;d73269e94d72:42927 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,169 DEBUG [RS:0;d73269e94d72:39133 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:45:31,169 DEBUG [RS:1;d73269e94d72:42927 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,169 DEBUG [RS:1;d73269e94d72:42927 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:31,169 DEBUG [RS:1;d73269e94d72:42927 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:45:31,170 DEBUG [RS:1;d73269e94d72:42927 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:45:31,172 INFO [RS:0;d73269e94d72:39133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,172 INFO [RS:1;d73269e94d72:42927 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,172 INFO [RS:0;d73269e94d72:39133 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,172 INFO [RS:1;d73269e94d72:42927 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,172 INFO [RS:0;d73269e94d72:39133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,172 INFO [RS:1;d73269e94d72:42927 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,172 INFO [RS:0;d73269e94d72:39133 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,172 INFO [RS:1;d73269e94d72:42927 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,172 INFO [RS:0;d73269e94d72:39133 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,172 INFO [RS:1;d73269e94d72:42927 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,172 INFO [RS:0;d73269e94d72:39133 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,39133,1732506329119-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:45:31,173 INFO [RS:1;d73269e94d72:42927 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,42927,1732506329341-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:45:31,191 INFO [RS:1;d73269e94d72:42927 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T03:45:31,194 INFO [RS:1;d73269e94d72:42927 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,42927,1732506329341-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,194 INFO [RS:1;d73269e94d72:42927 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,195 INFO [RS:1;d73269e94d72:42927 {}] regionserver.Replication(171): d73269e94d72,42927,1732506329341 started 2024-11-25T03:45:31,196 INFO [RS:0;d73269e94d72:39133 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T03:45:31,196 INFO [RS:0;d73269e94d72:39133 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,39133,1732506329119-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,197 INFO [RS:0;d73269e94d72:39133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,197 INFO [RS:0;d73269e94d72:39133 {}] regionserver.Replication(171): d73269e94d72,39133,1732506329119 started 2024-11-25T03:45:31,214 INFO [RS:1;d73269e94d72:42927 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,214 INFO [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(1482): Serving as d73269e94d72,42927,1732506329341, RpcServer on d73269e94d72/172.17.0.2:42927, sessionid=0x101701895360002 2024-11-25T03:45:31,215 DEBUG [RS:1;d73269e94d72:42927 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T03:45:31,216 DEBUG [RS:1;d73269e94d72:42927 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d73269e94d72,42927,1732506329341 2024-11-25T03:45:31,216 DEBUG [RS:1;d73269e94d72:42927 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,42927,1732506329341' 2024-11-25T03:45:31,216 DEBUG [RS:1;d73269e94d72:42927 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T03:45:31,218 DEBUG [RS:1;d73269e94d72:42927 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T03:45:31,218 DEBUG [RS:1;d73269e94d72:42927 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T03:45:31,219 DEBUG [RS:1;d73269e94d72:42927 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T03:45:31,219 DEBUG [RS:1;d73269e94d72:42927 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d73269e94d72,42927,1732506329341 2024-11-25T03:45:31,219 DEBUG [RS:1;d73269e94d72:42927 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,42927,1732506329341' 2024-11-25T03:45:31,219 DEBUG [RS:1;d73269e94d72:42927 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T03:45:31,220 DEBUG [RS:1;d73269e94d72:42927 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T03:45:31,221 INFO [RS:0;d73269e94d72:39133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,221 INFO [RS:0;d73269e94d72:39133 {}] regionserver.HRegionServer(1482): Serving as d73269e94d72,39133,1732506329119, RpcServer on d73269e94d72/172.17.0.2:39133, sessionid=0x101701895360001 2024-11-25T03:45:31,221 DEBUG [RS:1;d73269e94d72:42927 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T03:45:31,222 INFO [RS:1;d73269e94d72:42927 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T03:45:31,222 DEBUG [RS:0;d73269e94d72:39133 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T03:45:31,222 DEBUG [RS:0;d73269e94d72:39133 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d73269e94d72,39133,1732506329119 2024-11-25T03:45:31,222 INFO [RS:1;d73269e94d72:42927 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T03:45:31,222 DEBUG [RS:0;d73269e94d72:39133 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,39133,1732506329119' 2024-11-25T03:45:31,222 DEBUG [RS:0;d73269e94d72:39133 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T03:45:31,223 DEBUG [RS:0;d73269e94d72:39133 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T03:45:31,223 DEBUG [RS:0;d73269e94d72:39133 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T03:45:31,223 DEBUG [RS:0;d73269e94d72:39133 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T03:45:31,223 DEBUG [RS:0;d73269e94d72:39133 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d73269e94d72,39133,1732506329119 2024-11-25T03:45:31,224 DEBUG [RS:0;d73269e94d72:39133 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,39133,1732506329119' 2024-11-25T03:45:31,224 DEBUG [RS:0;d73269e94d72:39133 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T03:45:31,224 DEBUG [RS:0;d73269e94d72:39133 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T03:45:31,225 DEBUG [RS:0;d73269e94d72:39133 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T03:45:31,225 INFO [RS:0;d73269e94d72:39133 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T03:45:31,225 INFO [RS:0;d73269e94d72:39133 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T03:45:31,326 INFO [RS:1;d73269e94d72:42927 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:45:31,327 INFO [RS:0;d73269e94d72:39133 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:45:31,328 INFO [RS:0;d73269e94d72:39133 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=d73269e94d72%2C39133%2C1732506329119, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,39133,1732506329119, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/oldWALs, maxLogs=55 2024-11-25T03:45:31,328 INFO [RS:1;d73269e94d72:42927 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=d73269e94d72%2C42927%2C1732506329341, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,42927,1732506329341, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/oldWALs, maxLogs=55 2024-11-25T03:45:31,350 INFO [RS:1;d73269e94d72:42927 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,42927,1732506329341/d73269e94d72%2C42927%2C1732506329341.1732506331332 2024-11-25T03:45:31,351 DEBUG [RS:1;d73269e94d72:42927 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T03:45:31,354 INFO [RS:0;d73269e94d72:39133 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,39133,1732506329119/d73269e94d72%2C39133%2C1732506329119.1732506331332 2024-11-25T03:45:31,356 DEBUG [RS:0;d73269e94d72:39133 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T03:45:31,387 DEBUG [d73269e94d72:45215 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-25T03:45:31,398 DEBUG [d73269e94d72:45215 {}] balancer.BalancerClusterState(204): Hosts are {d73269e94d72=0} racks are {/default-rack=0} 2024-11-25T03:45:31,402 DEBUG [d73269e94d72:45215 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T03:45:31,402 DEBUG [d73269e94d72:45215 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T03:45:31,402 DEBUG [d73269e94d72:45215 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T03:45:31,402 DEBUG [d73269e94d72:45215 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T03:45:31,402 INFO [d73269e94d72:45215 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T03:45:31,402 INFO [d73269e94d72:45215 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T03:45:31,402 DEBUG [d73269e94d72:45215 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T03:45:31,410 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d73269e94d72,42927,1732506329341 2024-11-25T03:45:31,417 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,42927,1732506329341, state=OPENING 2024-11-25T03:45:31,435 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T03:45:31,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:31,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:31,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:31,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:31,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:31,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:31,449 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:45:31,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,42927,1732506329341}] 2024-11-25T03:45:31,626 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:45:31,629 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60591, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:45:31,636 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T03:45:31,636 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:45:31,637 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T03:45:31,638 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=d73269e94d72%2C42927%2C1732506329341.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,42927,1732506329341, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/oldWALs, maxLogs=55 2024-11-25T03:45:31,658 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,42927,1732506329341/d73269e94d72%2C42927%2C1732506329341.meta.1732506331640.meta 2024-11-25T03:45:31,658 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T03:45:31,658 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:45:31,660 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T03:45:31,663 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T03:45:31,668 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T03:45:31,672 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T03:45:31,673 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:45:31,673 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T03:45:31,674 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T03:45:31,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T03:45:31,677 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:45:31,677 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:31,677 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:31,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T03:45:31,678 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:45:31,679 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:31,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:31,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T03:45:31,680 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:45:31,680 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:31,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:31,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T03:45:31,681 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:45:31,681 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:31,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:31,681 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:45:31,683 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740 2024-11-25T03:45:31,684 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740 2024-11-25T03:45:31,685 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:45:31,685 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:45:31,685 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:45:31,687 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:45:31,688 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70918341, jitterRate=0.05676563084125519}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:45:31,688 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T03:45:31,690 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732506331674Writing region info on filesystem at 1732506331675 (+1 ms)Initializing all the Stores at 1732506331675Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506331675Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506331675Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506331675Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506331675Cleaning up temporary data from old regions at 1732506331685 (+10 ms)Running coprocessor post-open hooks at 1732506331688 (+3 ms)Region opened successfully at 1732506331690 (+2 ms) 2024-11-25T03:45:31,695 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732506331616 2024-11-25T03:45:31,704 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T03:45:31,704 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T03:45:31,705 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,42927,1732506329341 2024-11-25T03:45:31,706 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,42927,1732506329341, state=OPEN 2024-11-25T03:45:31,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:45:31,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:45:31,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:45:31,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:31,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:31,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:31,720 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d73269e94d72,42927,1732506329341 2024-11-25T03:45:31,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T03:45:31,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,42927,1732506329341 in 270 msec 2024-11-25T03:45:31,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T03:45:31,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 753 msec 2024-11-25T03:45:31,732 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:45:31,732 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T03:45:31,753 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:45:31,754 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,42927,1732506329341, seqNum=-1] 2024-11-25T03:45:31,770 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:45:31,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48311, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:45:31,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0420 sec 2024-11-25T03:45:31,794 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732506331794, completionTime=-1 2024-11-25T03:45:31,798 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-25T03:45:31,798 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:45:31,823 INFO [master/d73269e94d72:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:45:31,824 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506391824 2024-11-25T03:45:31,824 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506451824 2024-11-25T03:45:31,824 INFO [master/d73269e94d72:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-25T03:45:31,827 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:45:31,834 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,45215,1732506327714-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,835 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,45215,1732506327714-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,835 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,45215,1732506327714-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,837 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d73269e94d72:45215, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,837 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,838 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,845 DEBUG [master/d73269e94d72:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T03:45:31,873 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.199sec 2024-11-25T03:45:31,875 INFO [master/d73269e94d72:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T03:45:31,876 INFO [master/d73269e94d72:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T03:45:31,877 INFO [master/d73269e94d72:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T03:45:31,878 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T03:45:31,878 INFO [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T03:45:31,879 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,45215,1732506327714-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:45:31,880 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,45215,1732506327714-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T03:45:31,881 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T03:45:31,882 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T03:45:31,883 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,45215,1732506327714-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:31,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8eba43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:45:31,944 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T03:45:31,944 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T03:45:31,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d73269e94d72,45215,-1 for getting cluster id 2024-11-25T03:45:31,952 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:45:31,961 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '56a01f60-e64f-4f65-a758-996fad6bc065' 2024-11-25T03:45:31,965 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:45:31,965 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "56a01f60-e64f-4f65-a758-996fad6bc065" 2024-11-25T03:45:31,966 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328e3a0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:45:31,966 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d73269e94d72,45215,-1] 2024-11-25T03:45:31,969 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:45:31,972 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:31,975 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:45:31,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@201cd73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:45:31,979 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:45:31,988 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,42927,1732506329341, seqNum=-1] 2024-11-25T03:45:31,989 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:45:31,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:45:32,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d73269e94d72,45215,1732506327714 2024-11-25T03:45:32,027 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:45:32,034 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d73269e94d72,45215,1732506327714 2024-11-25T03:45:32,036 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@748b0a2d 2024-11-25T03:45:32,037 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:45:32,040 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:45:32,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45215 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732506332024', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:45:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45215 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732506332024 2024-11-25T03:45:32,052 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732506332024 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:45:32,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45215 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732506332024" procId is: 4 2024-11-25T03:45:32,055 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:32,056 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732506332024 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:45:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:45:32,068 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732506332024-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7c407e6099ea24796fec42b4ec03daf1, NAME => 'testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732506332024', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1 2024-11-25T03:45:32,074 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732506332024-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:45:32,074 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732506332024-pool-0 {}] regionserver.HRegion(1722): Closing 7c407e6099ea24796fec42b4ec03daf1, disabling compactions & flushes 2024-11-25T03:45:32,075 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732506332024-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:32,075 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732506332024-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:32,075 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732506332024-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. after waiting 0 ms 2024-11-25T03:45:32,075 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732506332024-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:32,075 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732506332024-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:32,075 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732506332024-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7c407e6099ea24796fec42b4ec03daf1: Waiting for close lock at 1732506332074Disabling compacts and flushes for region at 1732506332074Disabling writes for close at 1732506332075 (+1 ms)Writing region close event to WAL at 1732506332075Closed at 1732506332075 2024-11-25T03:45:32,076 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732506332024 execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:45:32,080 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732506332076"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506332076"}]},"ts":"1732506332076"} 2024-11-25T03:45:32,084 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:45:32,085 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732506332024 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:45:32,088 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732506332024","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732506332086"}]},"ts":"1732506332086"} 2024-11-25T03:45:32,091 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732506332024, state=ENABLING in hbase:meta 2024-11-25T03:45:32,092 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {d73269e94d72=0} racks are {/default-rack=0} 2024-11-25T03:45:32,092 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T03:45:32,092 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T03:45:32,092 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T03:45:32,092 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T03:45:32,092 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T03:45:32,092 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T03:45:32,093 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T03:45:32,094 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732506332024, region=7c407e6099ea24796fec42b4ec03daf1, ASSIGN}] 2024-11-25T03:45:32,096 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732506332024, region=7c407e6099ea24796fec42b4ec03daf1, ASSIGN 2024-11-25T03:45:32,097 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732506332024, region=7c407e6099ea24796fec42b4ec03daf1, ASSIGN; state=OFFLINE, location=d73269e94d72,39133,1732506329119; forceNewPlan=false, retain=false 2024-11-25T03:45:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:45:32,250 INFO [d73269e94d72:45215 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T03:45:32,252 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7c407e6099ea24796fec42b4ec03daf1, regionState=OPENING, regionLocation=d73269e94d72,39133,1732506329119 2024-11-25T03:45:32,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732506332024, region=7c407e6099ea24796fec42b4ec03daf1, ASSIGN because future has completed 2024-11-25T03:45:32,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c407e6099ea24796fec42b4ec03daf1, server=d73269e94d72,39133,1732506329119}] 2024-11-25T03:45:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:45:32,413 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:45:32,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51015, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:45:32,419 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:32,420 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c407e6099ea24796fec42b4ec03daf1, NAME => 'testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:45:32,421 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732506332024 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:32,421 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:45:32,421 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:32,421 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:32,422 INFO [StoreOpener-7c407e6099ea24796fec42b4ec03daf1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:32,426 INFO [StoreOpener-7c407e6099ea24796fec42b4ec03daf1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c407e6099ea24796fec42b4ec03daf1 columnFamilyName family 2024-11-25T03:45:32,426 DEBUG [StoreOpener-7c407e6099ea24796fec42b4ec03daf1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:32,427 INFO [StoreOpener-7c407e6099ea24796fec42b4ec03daf1-1 {}] regionserver.HStore(327): Store=7c407e6099ea24796fec42b4ec03daf1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:45:32,427 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:32,429 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:32,429 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:32,429 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:32,429 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:32,430 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:32,436 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:45:32,436 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7c407e6099ea24796fec42b4ec03daf1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75377883, jitterRate=0.12321798503398895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:45:32,437 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:32,438 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7c407e6099ea24796fec42b4ec03daf1: Running coprocessor pre-open hook at 1732506332421Writing region info on filesystem at 1732506332421Initializing all the Stores at 1732506332422 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506332422Cleaning up temporary data from old regions at 1732506332429 (+7 ms)Running coprocessor post-open hooks at 1732506332437 (+8 ms)Region opened successfully at 1732506332438 (+1 ms) 2024-11-25T03:45:32,439 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1., pid=6, masterSystemTime=1732506332413 2024-11-25T03:45:32,442 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:32,442 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:32,442 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7c407e6099ea24796fec42b4ec03daf1, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,39133,1732506329119 2024-11-25T03:45:32,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c407e6099ea24796fec42b4ec03daf1, server=d73269e94d72,39133,1732506329119 because future has completed 2024-11-25T03:45:32,448 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T03:45:32,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c407e6099ea24796fec42b4ec03daf1, server=d73269e94d72,39133,1732506329119 in 187 msec 2024-11-25T03:45:32,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T03:45:32,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732506332024, region=7c407e6099ea24796fec42b4ec03daf1, ASSIGN in 355 msec 2024-11-25T03:45:32,452 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732506332024 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:45:32,452 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732506332024","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732506332452"}]},"ts":"1732506332452"} 2024-11-25T03:45:32,455 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732506332024, state=ENABLED in hbase:meta 2024-11-25T03:45:32,456 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732506332024 execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:45:32,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732506332024 in 409 msec 2024-11-25T03:45:32,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:45:32,685 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732506332024 get assigned. Timeout = 60000ms 2024-11-25T03:45:32,685 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732506332024 completed 2024-11-25T03:45:32,688 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:45:32,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732506332024 assigned to meta. Checking AM states. 2024-11-25T03:45:32,694 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:45:32,695 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732506332024 assigned. 2024-11-25T03:45:32,702 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732506332024', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1., hostname=d73269e94d72,39133,1732506329119, seqNum=2] 2024-11-25T03:45:32,704 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:45:32,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:45:32,716 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7c407e6099ea24796fec42b4ec03daf1 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-25T03:45:32,737 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f5da399eec354b6093aa8910f12961f5 2024-11-25T03:45:32,770 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/.tmp/family/f5da399eec354b6093aa8910f12961f5 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-25T03:45:32,775 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/.tmp/family/f5da399eec354b6093aa8910f12961f5 2024-11-25T03:45:32,813 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/.tmp/family/f5da399eec354b6093aa8910f12961f5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/family/f5da399eec354b6093aa8910f12961f5 2024-11-25T03:45:32,815 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/family/f5da399eec354b6093aa8910f12961f5, delay=995 ms 2024-11-25T03:45:32,818 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/family/f5da399eec354b6093aa8910f12961f5, entries=2, sequenceid=6, filesize=4.9 K 2024-11-25T03:45:32,825 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 7c407e6099ea24796fec42b4ec03daf1 in 108ms, sequenceid=6, compaction requested=false 2024-11-25T03:45:32,826 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732506332024' 2024-11-25T03:45:32,828 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7c407e6099ea24796fec42b4ec03daf1: 2024-11-25T03:45:33,813 DEBUG [hfile-prefetch-1732506332815 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f5da399eec354b6093aa8910f12961f5_0. 2024-11-25T03:45:33,820 DEBUG [hfile-prefetch-1732506332815 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f5da399eec354b6093aa8910f12961f5_117. 2024-11-25T03:45:33,822 DEBUG [hfile-prefetch-1732506332815 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/family/f5da399eec354b6093aa8910f12961f5, with total blocks 2, and data blocks 1 2024-11-25T03:45:33,822 DEBUG [hfile-prefetch-1732506332815 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-25T03:45:33,823 DEBUG [hfile-prefetch-1732506332815 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/family/f5da399eec354b6093aa8910f12961f5, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-25T03:45:33,823 DEBUG [hfile-prefetch-1732506332815 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/family/f5da399eec354b6093aa8910f12961f5. Will sleep 100ms and try the verification again. 2024-11-25T03:45:33,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45215 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=7c407e6099ea24796fec42b4ec03daf1, source=d73269e94d72,39133,1732506329119, destination=d73269e94d72,42927,1732506329341, warming up region on d73269e94d72,42927,1732506329341 2024-11-25T03:45:33,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45215 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=7c407e6099ea24796fec42b4ec03daf1, source=d73269e94d72,39133,1732506329119, destination=d73269e94d72,42927,1732506329341, running balancer 2024-11-25T03:45:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45215 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732506332024, region=7c407e6099ea24796fec42b4ec03daf1, REOPEN/MOVE 2024-11-25T03:45:33,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] regionserver.HRegion(7855): Warmup {ENCODED => 7c407e6099ea24796fec42b4ec03daf1, NAME => 'testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:45:33,846 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732506332024, region=7c407e6099ea24796fec42b4ec03daf1, REOPEN/MOVE 2024-11-25T03:45:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:45:33,850 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7c407e6099ea24796fec42b4ec03daf1, regionState=CLOSING, regionLocation=d73269e94d72,39133,1732506329119 2024-11-25T03:45:33,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732506332024, region=7c407e6099ea24796fec42b4ec03daf1, REOPEN/MOVE because future has completed 2024-11-25T03:45:33,856 INFO [StoreOpener-7c407e6099ea24796fec42b4ec03daf1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:33,857 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-25T03:45:33,857 INFO [StoreOpener-7c407e6099ea24796fec42b4ec03daf1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c407e6099ea24796fec42b4ec03daf1 columnFamilyName family 2024-11-25T03:45:33,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7c407e6099ea24796fec42b4ec03daf1, server=d73269e94d72,39133,1732506329119}] 2024-11-25T03:45:33,858 DEBUG [StoreOpener-7c407e6099ea24796fec42b4ec03daf1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:33,866 DEBUG [StoreOpener-7c407e6099ea24796fec42b4ec03daf1-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/family/f5da399eec354b6093aa8910f12961f5 2024-11-25T03:45:33,866 INFO [StoreOpener-7c407e6099ea24796fec42b4ec03daf1-1 {}] regionserver.HStore(327): Store=7c407e6099ea24796fec42b4ec03daf1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:45:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] regionserver.HRegion(1722): Closing 7c407e6099ea24796fec42b4ec03daf1, disabling compactions & flushes 2024-11-25T03:45:33,866 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. after waiting 0 ms 2024-11-25T03:45:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:33,868 DEBUG [StoreFileCloser-7c407e6099ea24796fec42b4ec03daf1-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/family/f5da399eec354b6093aa8910f12961f5 2024-11-25T03:45:33,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42927 {}] regionserver.HRegion(1676): Region close journal for 7c407e6099ea24796fec42b4ec03daf1: Waiting for close lock at 1732506333866Disabling compacts and flushes for region at 1732506333866Disabling writes for close at 1732506333867 (+1 ms)Writing region close event to WAL at 1732506333869 (+2 ms)Closed at 1732506333869 2024-11-25T03:45:33,871 WARN [hfile-prefetch-1732506332815 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/family/f5da399eec354b6093aa8910f12961f5, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-25T03:45:33,871 DEBUG [hfile-prefetch-1732506332815 {}] hfile.PrefetchExecutor(121): Prefetch completed for f5da399eec354b6093aa8910f12961f5 2024-11-25T03:45:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45215 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-25T03:45:33,993 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T03:45:33,993 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T03:45:33,994 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T03:45:33,994 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T03:45:34,016 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T03:45:34,016 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-25T03:45:34,016 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T03:45:34,016 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732506333993 2024-11-25T03:45:34,016 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T03:45:34,016 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T03:45:34,016 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T03:45:34,024 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T03:45:34,024 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732506334016 2024-11-25T03:45:34,024 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T03:45:34,024 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:34,025 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-25T03:45:34,025 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 7c407e6099ea24796fec42b4ec03daf1, disabling compactions & flushes 2024-11-25T03:45:34,026 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:34,026 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:34,026 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. after waiting 0 ms 2024-11-25T03:45:34,026 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:34,028 DEBUG [StoreFileCloser-7c407e6099ea24796fec42b4ec03daf1-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file f5da399eec354b6093aa8910f12961f5, starting offset: 0, end offset: 9223372036854775807 2024-11-25T03:45:34,030 DEBUG [StoreFileCloser-7c407e6099ea24796fec42b4ec03daf1-family-1 {}] bucket.BucketCache(856): removed key f5da399eec354b6093aa8910f12961f5_0 from back map with offset lock 2094080 in the evict process 2024-11-25T03:45:34,049 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-25T03:45:34,050 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:34,050 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 7c407e6099ea24796fec42b4ec03daf1: Waiting for close lock at 1732506334025Running coprocessor pre-close hooks at 1732506334025Disabling compacts and flushes for region at 1732506334025Disabling writes for close at 1732506334026 (+1 ms)Writing region close event to WAL at 1732506334031 (+5 ms)Running coprocessor post-close hooks at 1732506334049 (+18 ms)Closed at 1732506334050 (+1 ms) 2024-11-25T03:45:34,050 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 7c407e6099ea24796fec42b4ec03daf1 move to d73269e94d72,42927,1732506329341 record at close sequenceid=6 2024-11-25T03:45:34,054 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:34,054 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7c407e6099ea24796fec42b4ec03daf1, regionState=CLOSED 2024-11-25T03:45:34,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7c407e6099ea24796fec42b4ec03daf1, server=d73269e94d72,39133,1732506329119 because future has completed 2024-11-25T03:45:34,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T03:45:34,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 7c407e6099ea24796fec42b4ec03daf1, server=d73269e94d72,39133,1732506329119 in 203 msec 2024-11-25T03:45:34,064 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732506332024, region=7c407e6099ea24796fec42b4ec03daf1, REOPEN/MOVE; state=CLOSED, location=d73269e94d72,42927,1732506329341; forceNewPlan=false, retain=false 2024-11-25T03:45:34,116 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T03:45:34,116 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T03:45:34,117 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T03:45:34,117 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T03:45:34,124 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T03:45:34,124 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T03:45:34,124 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732506334116 2024-11-25T03:45:34,124 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T03:45:34,124 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T03:45:34,124 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T03:45:34,124 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T03:45:34,133 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T03:45:34,133 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732506334124 2024-11-25T03:45:34,133 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T03:45:34,215 INFO [d73269e94d72:45215 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T03:45:34,215 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7c407e6099ea24796fec42b4ec03daf1, regionState=OPENING, regionLocation=d73269e94d72,42927,1732506329341 2024-11-25T03:45:34,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732506332024, region=7c407e6099ea24796fec42b4ec03daf1, REOPEN/MOVE because future has completed 2024-11-25T03:45:34,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c407e6099ea24796fec42b4ec03daf1, server=d73269e94d72,42927,1732506329341}] 2024-11-25T03:45:34,378 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:34,379 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c407e6099ea24796fec42b4ec03daf1, NAME => 'testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:45:34,380 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732506332024 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:34,380 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:45:34,380 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:34,380 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:34,382 INFO [StoreOpener-7c407e6099ea24796fec42b4ec03daf1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:34,383 INFO [StoreOpener-7c407e6099ea24796fec42b4ec03daf1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c407e6099ea24796fec42b4ec03daf1 columnFamilyName family 2024-11-25T03:45:34,383 DEBUG [StoreOpener-7c407e6099ea24796fec42b4ec03daf1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:34,386 DEBUG [StoreFileOpener-7c407e6099ea24796fec42b4ec03daf1-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/family/f5da399eec354b6093aa8910f12961f5, delay=962 ms 2024-11-25T03:45:34,388 DEBUG [StoreOpener-7c407e6099ea24796fec42b4ec03daf1-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/family/f5da399eec354b6093aa8910f12961f5 2024-11-25T03:45:34,388 INFO [StoreOpener-7c407e6099ea24796fec42b4ec03daf1-1 {}] regionserver.HStore(327): Store=7c407e6099ea24796fec42b4ec03daf1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:45:34,388 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:34,389 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:34,389 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:34,390 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:34,390 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:34,394 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:34,394 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 7c407e6099ea24796fec42b4ec03daf1; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59859034, jitterRate=-0.10803088545799255}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:45:34,394 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:34,395 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 7c407e6099ea24796fec42b4ec03daf1: Running coprocessor pre-open hook at 1732506334381Writing region info on filesystem at 1732506334381Initializing all the Stores at 1732506334381Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506334381Cleaning up temporary data from old regions at 1732506334390 (+9 ms)Running coprocessor post-open hooks at 1732506334394 (+4 ms)Region opened successfully at 1732506334395 (+1 ms) 2024-11-25T03:45:34,396 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1., pid=9, masterSystemTime=1732506334371 2024-11-25T03:45:34,399 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:34,399 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:34,399 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7c407e6099ea24796fec42b4ec03daf1, regionState=OPEN, openSeqNum=10, regionLocation=d73269e94d72,42927,1732506329341 2024-11-25T03:45:34,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c407e6099ea24796fec42b4ec03daf1, server=d73269e94d72,42927,1732506329341 because future has completed 2024-11-25T03:45:34,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-25T03:45:34,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c407e6099ea24796fec42b4ec03daf1, server=d73269e94d72,42927,1732506329341 in 185 msec 2024-11-25T03:45:34,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732506332024, region=7c407e6099ea24796fec42b4ec03daf1, REOPEN/MOVE in 561 msec 2024-11-25T03:45:34,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T03:45:34,451 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:45:34,452 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:45:34,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:34,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:34,459 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:45:34,459 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T03:45:34,459 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=377653908, stopped=false 2024-11-25T03:45:34,460 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d73269e94d72,45215,1732506327714 2024-11-25T03:45:34,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:45:34,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:45:34,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:45:34,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:34,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:34,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:34,542 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:45:34,543 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:45:34,543 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:45:34,543 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:45:34,544 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:45:34,544 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:45:34,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:34,545 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd73269e94d72,39133,1732506329119' ***** 2024-11-25T03:45:34,546 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T03:45:34,546 INFO [RS:0;d73269e94d72:39133 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T03:45:34,546 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd73269e94d72,42927,1732506329341' ***** 2024-11-25T03:45:34,546 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T03:45:34,547 INFO [RS:0;d73269e94d72:39133 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/bucket.persistence 2024-11-25T03:45:34,547 INFO [RS:1;d73269e94d72:42927 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T03:45:34,547 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T03:45:34,547 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T03:45:34,547 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T03:45:34,547 INFO [RS:1;d73269e94d72:42927 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/bucket.persistence 2024-11-25T03:45:34,547 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T03:45:34,548 INFO [RS:0;d73269e94d72:39133 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T03:45:34,548 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T03:45:34,548 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T03:45:34,548 INFO [RS:1;d73269e94d72:42927 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T03:45:34,548 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T03:45:34,549 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T03:45:34,558 DEBUG [RS:0;d73269e94d72:39133 {}] bucket.BucketCache(1487): Thread RS:0;d73269e94d72:39133 started persisting bucket cache to file 2024-11-25T03:45:34,558 DEBUG [RS:0;d73269e94d72:39133 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T03:45:34,558 DEBUG [RS:0;d73269e94d72:39133 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T03:45:34,558 DEBUG [RS:1;d73269e94d72:42927 {}] bucket.BucketCache(1487): Thread RS:1;d73269e94d72:42927 started persisting bucket cache to file 2024-11-25T03:45:34,558 DEBUG [RS:1;d73269e94d72:42927 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T03:45:34,558 DEBUG [RS:1;d73269e94d72:42927 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T03:45:34,561 DEBUG [RS:0;d73269e94d72:39133 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T03:45:34,561 DEBUG [RS:0;d73269e94d72:39133 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732506334558 2024-11-25T03:45:34,561 DEBUG [RS:0;d73269e94d72:39133 {}] bucket.BucketCache(1509): Thread RS:0;d73269e94d72:39133 finished persisting bucket cache to file, renaming 2024-11-25T03:45:34,561 INFO [RS:0;d73269e94d72:39133 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T03:45:34,562 INFO [RS:0;d73269e94d72:39133 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T03:45:34,562 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T03:45:34,562 INFO [RS:0;d73269e94d72:39133 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T03:45:34,562 INFO [RS:0;d73269e94d72:39133 {}] regionserver.HRegionServer(959): stopping server d73269e94d72,39133,1732506329119 2024-11-25T03:45:34,562 INFO [RS:0;d73269e94d72:39133 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:45:34,562 INFO [RS:0;d73269e94d72:39133 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d73269e94d72:39133. 2024-11-25T03:45:34,562 DEBUG [RS:0;d73269e94d72:39133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:45:34,562 DEBUG [RS:0;d73269e94d72:39133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:34,562 INFO [RS:0;d73269e94d72:39133 {}] regionserver.HRegionServer(976): stopping server d73269e94d72,39133,1732506329119; all regions closed. 2024-11-25T03:45:34,567 WARN [Thread-97 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedReader.ensureOpen(BufferedReader.java:123) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:321) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-25T03:45:34,567 ERROR [RS:1;d73269e94d72:42927 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:45:34,568 DEBUG [RS:1;d73269e94d72:42927 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T03:45:34,568 DEBUG [RS:1;d73269e94d72:42927 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732506334558 2024-11-25T03:45:34,568 DEBUG [RS:1;d73269e94d72:42927 {}] bucket.BucketCache(1509): Thread RS:1;d73269e94d72:42927 finished persisting bucket cache to file, renaming 2024-11-25T03:45:34,568 WARN [RS:1;d73269e94d72:42927 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-25T03:45:34,568 INFO [RS:1;d73269e94d72:42927 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T03:45:34,568 INFO [RS:1;d73269e94d72:42927 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T03:45:34,568 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T03:45:34,569 INFO [RS:1;d73269e94d72:42927 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T03:45:34,569 INFO [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(3091): Received CLOSE for 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:34,570 INFO [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(959): stopping server d73269e94d72,42927,1732506329341 2024-11-25T03:45:34,570 INFO [RS:1;d73269e94d72:42927 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:45:34,570 INFO [RS:1;d73269e94d72:42927 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;d73269e94d72:42927. 2024-11-25T03:45:34,570 DEBUG [RS:1;d73269e94d72:42927 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:45:34,570 DEBUG [RS:1;d73269e94d72:42927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:34,570 INFO [RS:1;d73269e94d72:42927 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T03:45:34,571 INFO [RS:1;d73269e94d72:42927 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T03:45:34,571 INFO [RS:1;d73269e94d72:42927 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T03:45:34,571 INFO [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T03:45:34,571 INFO [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-25T03:45:34,571 DEBUG [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 7c407e6099ea24796fec42b4ec03daf1=testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1.} 2024-11-25T03:45:34,572 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T03:45:34,572 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T03:45:34,572 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7c407e6099ea24796fec42b4ec03daf1, disabling compactions & flushes 2024-11-25T03:45:34,572 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:45:34,572 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:34,572 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:45:34,572 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:34,572 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:45:34,572 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. after waiting 0 ms 2024-11-25T03:45:34,572 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:45:34,572 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:34,572 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:45:34,573 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-25T03:45:34,575 DEBUG [StoreFileCloser-7c407e6099ea24796fec42b4ec03daf1-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/family/f5da399eec354b6093aa8910f12961f5 2024-11-25T03:45:34,575 DEBUG [RS:0;d73269e94d72:39133 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/oldWALs 2024-11-25T03:45:34,575 INFO [RS:0;d73269e94d72:39133 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d73269e94d72%2C39133%2C1732506329119:(num 1732506331332) 2024-11-25T03:45:34,575 DEBUG [RS:0;d73269e94d72:39133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:34,575 INFO [RS:0;d73269e94d72:39133 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:45:34,575 INFO [RS:0;d73269e94d72:39133 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:45:34,576 DEBUG [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7c407e6099ea24796fec42b4ec03daf1 2024-11-25T03:45:34,576 INFO [regionserver/d73269e94d72:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:45:34,576 INFO [regionserver/d73269e94d72:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:45:34,576 INFO [RS:0;d73269e94d72:39133 {}] hbase.ChoreService(370): Chore service for: regionserver/d73269e94d72:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T03:45:34,576 INFO [regionserver/d73269e94d72:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:45:34,576 INFO [RS:0;d73269e94d72:39133 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T03:45:34,576 INFO [RS:0;d73269e94d72:39133 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T03:45:34,577 INFO [RS:0;d73269e94d72:39133 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T03:45:34,577 INFO [RS:0;d73269e94d72:39133 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:45:34,577 INFO [RS:0;d73269e94d72:39133 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39133 2024-11-25T03:45:34,581 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnRegionMove1732506332024/7c407e6099ea24796fec42b4ec03daf1/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-25T03:45:34,581 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:34,581 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7c407e6099ea24796fec42b4ec03daf1: Waiting for close lock at 1732506334571Running coprocessor pre-close hooks at 1732506334572 (+1 ms)Disabling compacts and flushes for region at 1732506334572Disabling writes for close at 1732506334572Writing region close event to WAL at 1732506334575 (+3 ms)Running coprocessor post-close hooks at 1732506334581 (+6 ms)Closed at 1732506334581 2024-11-25T03:45:34,582 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1. 2024-11-25T03:45:34,585 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8cd9804b2f59483eb7126c8883484eea 2024-11-25T03:45:34,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:45:34,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d73269e94d72,39133,1732506329119 2024-11-25T03:45:34,593 INFO [RS:0;d73269e94d72:39133 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:45:34,594 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d73269e94d72,39133,1732506329119] 2024-11-25T03:45:34,606 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/info/8cd9804b2f59483eb7126c8883484eea is 205, key is testBlockEvictionOnRegionMove1732506332024,,1732506332041.7c407e6099ea24796fec42b4ec03daf1./info:regioninfo/1732506334399/Put/seqid=0 2024-11-25T03:45:34,607 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/info/8cd9804b2f59483eb7126c8883484eea 2024-11-25T03:45:34,614 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d73269e94d72,39133,1732506329119 already deleted, retry=false 2024-11-25T03:45:34,614 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d73269e94d72,39133,1732506329119 expired; onlineServers=1 2024-11-25T03:45:34,617 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d03bf444766e46b78c542ba417bd3973 2024-11-25T03:45:34,631 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/ns/d03bf444766e46b78c542ba417bd3973 is 43, key is default/ns:d/1732506331776/Put/seqid=0 2024-11-25T03:45:34,632 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/ns/d03bf444766e46b78c542ba417bd3973 2024-11-25T03:45:34,641 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1aae7ebe8f704e64a8f4cbb4b118fe2a 2024-11-25T03:45:34,652 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/table/1aae7ebe8f704e64a8f4cbb4b118fe2a is 78, key is testBlockEvictionOnRegionMove1732506332024/table:state/1732506332452/Put/seqid=0 2024-11-25T03:45:34,653 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/table/1aae7ebe8f704e64a8f4cbb4b118fe2a 2024-11-25T03:45:34,655 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/info/8cd9804b2f59483eb7126c8883484eea as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/info/8cd9804b2f59483eb7126c8883484eea 2024-11-25T03:45:34,656 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/info/8cd9804b2f59483eb7126c8883484eea, delay=902 ms 2024-11-25T03:45:34,658 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/info/8cd9804b2f59483eb7126c8883484eea, entries=15, sequenceid=15, filesize=7.6 K 2024-11-25T03:45:34,658 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/ns/d03bf444766e46b78c542ba417bd3973 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/ns/d03bf444766e46b78c542ba417bd3973 2024-11-25T03:45:34,659 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/ns/d03bf444766e46b78c542ba417bd3973, delay=952 ms 2024-11-25T03:45:34,661 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/ns/d03bf444766e46b78c542ba417bd3973, entries=2, sequenceid=15, filesize=5.0 K 2024-11-25T03:45:34,661 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/table/1aae7ebe8f704e64a8f4cbb4b118fe2a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/table/1aae7ebe8f704e64a8f4cbb4b118fe2a 2024-11-25T03:45:34,662 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/table/1aae7ebe8f704e64a8f4cbb4b118fe2a, delay=982 ms 2024-11-25T03:45:34,664 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/table/1aae7ebe8f704e64a8f4cbb4b118fe2a, entries=2, sequenceid=15, filesize=5.3 K 2024-11-25T03:45:34,664 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 92ms, sequenceid=15, compaction requested=false 2024-11-25T03:45:34,665 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T03:45:34,666 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/info/8cd9804b2f59483eb7126c8883484eea 2024-11-25T03:45:34,666 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/ns/d03bf444766e46b78c542ba417bd3973 2024-11-25T03:45:34,667 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/table/1aae7ebe8f704e64a8f4cbb4b118fe2a 2024-11-25T03:45:34,671 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-25T03:45:34,671 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:45:34,671 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:45:34,671 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732506334571Running coprocessor pre-close hooks at 1732506334572 (+1 ms)Disabling compacts and flushes for region at 1732506334572Disabling writes for close at 1732506334572Obtaining lock to block concurrent updates at 1732506334573 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732506334573Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732506334573Flushing stores of hbase:meta,,1.1588230740 at 1732506334574 (+1 ms)Flushing 1588230740/info: creating writer at 1732506334574Flushing 1588230740/info: appending metadata at 1732506334604 (+30 ms)Flushing 1588230740/info: closing flushed file at 1732506334604Flushing 1588230740/ns: creating writer at 1732506334611 (+7 ms)Flushing 1588230740/ns: appending metadata at 1732506334630 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732506334630Flushing 1588230740/table: creating writer at 1732506334635 (+5 ms)Flushing 1588230740/table: appending metadata at 1732506334651 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732506334651Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16cbe877: reopening flushed file at 1732506334655 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7eca2c0c: reopening flushed file at 1732506334658 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@103e565a: reopening flushed file at 1732506334661 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 92ms, sequenceid=15, compaction requested=false at 1732506334665 (+4 ms)Writing region close event to WAL at 1732506334667 (+2 ms)Running coprocessor post-close hooks at 1732506334671 (+4 ms)Closed at 1732506334671 2024-11-25T03:45:34,672 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T03:45:34,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:45:34,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39133-0x101701895360001, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:45:34,705 INFO [RS:0;d73269e94d72:39133 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:45:34,705 INFO [RS:0;d73269e94d72:39133 {}] regionserver.HRegionServer(1031): Exiting; stopping=d73269e94d72,39133,1732506329119; zookeeper connection closed. 2024-11-25T03:45:34,706 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3da8dfc4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3da8dfc4 2024-11-25T03:45:34,776 INFO [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(976): stopping server d73269e94d72,42927,1732506329341; all regions closed. 2024-11-25T03:45:34,779 DEBUG [RS:1;d73269e94d72:42927 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/oldWALs 2024-11-25T03:45:34,780 INFO [RS:1;d73269e94d72:42927 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d73269e94d72%2C42927%2C1732506329341.meta:.meta(num 1732506331640) 2024-11-25T03:45:34,782 DEBUG [RS:1;d73269e94d72:42927 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/oldWALs 2024-11-25T03:45:34,783 INFO [RS:1;d73269e94d72:42927 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d73269e94d72%2C42927%2C1732506329341:(num 1732506331332) 2024-11-25T03:45:34,783 DEBUG [RS:1;d73269e94d72:42927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:34,783 INFO [RS:1;d73269e94d72:42927 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:45:34,783 INFO [RS:1;d73269e94d72:42927 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:45:34,783 INFO [RS:1;d73269e94d72:42927 {}] hbase.ChoreService(370): Chore service for: regionserver/d73269e94d72:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T03:45:34,783 INFO [RS:1;d73269e94d72:42927 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:45:34,784 INFO [regionserver/d73269e94d72:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:45:34,784 INFO [RS:1;d73269e94d72:42927 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42927 2024-11-25T03:45:34,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d73269e94d72,42927,1732506329341 2024-11-25T03:45:34,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:45:34,793 INFO [RS:1;d73269e94d72:42927 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:45:34,803 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d73269e94d72,42927,1732506329341] 2024-11-25T03:45:34,814 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d73269e94d72,42927,1732506329341 already deleted, retry=false 2024-11-25T03:45:34,814 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d73269e94d72,42927,1732506329341 expired; onlineServers=0 2024-11-25T03:45:34,814 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd73269e94d72,45215,1732506327714' ***** 2024-11-25T03:45:34,814 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T03:45:34,814 INFO [M:0;d73269e94d72:45215 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:45:34,814 INFO [M:0;d73269e94d72:45215 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:45:34,815 DEBUG [M:0;d73269e94d72:45215 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T03:45:34,815 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T03:45:34,815 DEBUG [M:0;d73269e94d72:45215 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T03:45:34,815 DEBUG [master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.large.0-1732506330892 {}] cleaner.HFileCleaner(306): Exit Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.large.0-1732506330892,5,FailOnTimeoutGroup] 2024-11-25T03:45:34,815 DEBUG [master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.small.0-1732506330897 {}] cleaner.HFileCleaner(306): Exit Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.small.0-1732506330897,5,FailOnTimeoutGroup] 2024-11-25T03:45:34,815 INFO [M:0;d73269e94d72:45215 {}] hbase.ChoreService(370): Chore service for: master/d73269e94d72:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T03:45:34,815 INFO [M:0;d73269e94d72:45215 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:45:34,815 DEBUG [M:0;d73269e94d72:45215 {}] master.HMaster(1795): Stopping service threads 2024-11-25T03:45:34,815 INFO [M:0;d73269e94d72:45215 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T03:45:34,816 INFO [M:0;d73269e94d72:45215 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:45:34,816 INFO [M:0;d73269e94d72:45215 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:45:34,816 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T03:45:34,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T03:45:34,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:34,825 DEBUG [M:0;d73269e94d72:45215 {}] zookeeper.ZKUtil(347): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T03:45:34,825 WARN [M:0;d73269e94d72:45215 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T03:45:34,825 INFO [M:0;d73269e94d72:45215 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/.lastflushedseqids 2024-11-25T03:45:34,836 INFO [M:0;d73269e94d72:45215 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:45:34,837 INFO [M:0;d73269e94d72:45215 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T03:45:34,837 DEBUG [M:0;d73269e94d72:45215 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T03:45:34,837 INFO [M:0;d73269e94d72:45215 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:45:34,837 DEBUG [M:0;d73269e94d72:45215 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:45:34,837 DEBUG [M:0;d73269e94d72:45215 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T03:45:34,837 DEBUG [M:0;d73269e94d72:45215 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:45:34,837 INFO [M:0;d73269e94d72:45215 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-25T03:45:34,846 DEBUG [M:0;d73269e94d72:45215 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on da6a2d6257ee4f32be138ea67d5c18cd 2024-11-25T03:45:34,858 DEBUG [M:0;d73269e94d72:45215 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/da6a2d6257ee4f32be138ea67d5c18cd is 82, key is hbase:meta,,1/info:regioninfo/1732506331705/Put/seqid=0 2024-11-25T03:45:34,859 INFO [M:0;d73269e94d72:45215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/da6a2d6257ee4f32be138ea67d5c18cd 2024-11-25T03:45:34,866 DEBUG [M:0;d73269e94d72:45215 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 106ebe2149e74ea4ae4dbd3e30f01a21 2024-11-25T03:45:34,878 DEBUG [M:0;d73269e94d72:45215 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/106ebe2149e74ea4ae4dbd3e30f01a21 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732506332457/Put/seqid=0 2024-11-25T03:45:34,879 INFO [M:0;d73269e94d72:45215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/106ebe2149e74ea4ae4dbd3e30f01a21 2024-11-25T03:45:34,888 DEBUG [M:0;d73269e94d72:45215 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f0a1d3ce154d416094cc4dd691877758 2024-11-25T03:45:34,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:45:34,904 INFO [RS:1;d73269e94d72:42927 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:45:34,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42927-0x101701895360002, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:45:34,904 INFO [RS:1;d73269e94d72:42927 {}] regionserver.HRegionServer(1031): Exiting; stopping=d73269e94d72,42927,1732506329341; zookeeper connection closed. 2024-11-25T03:45:34,904 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@174c5066 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@174c5066 2024-11-25T03:45:34,905 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-25T03:45:34,906 DEBUG [M:0;d73269e94d72:45215 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f0a1d3ce154d416094cc4dd691877758 is 69, key is d73269e94d72,39133,1732506329119/rs:state/1732506331036/Put/seqid=0 2024-11-25T03:45:34,907 INFO [M:0;d73269e94d72:45215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f0a1d3ce154d416094cc4dd691877758 2024-11-25T03:45:34,910 DEBUG [M:0;d73269e94d72:45215 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/da6a2d6257ee4f32be138ea67d5c18cd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/da6a2d6257ee4f32be138ea67d5c18cd 2024-11-25T03:45:34,912 INFO [M:0;d73269e94d72:45215 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/da6a2d6257ee4f32be138ea67d5c18cd, entries=8, sequenceid=84, filesize=5.5 K 2024-11-25T03:45:34,912 DEBUG [M:0;d73269e94d72:45215 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/106ebe2149e74ea4ae4dbd3e30f01a21 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/106ebe2149e74ea4ae4dbd3e30f01a21 2024-11-25T03:45:34,915 INFO [M:0;d73269e94d72:45215 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/106ebe2149e74ea4ae4dbd3e30f01a21, entries=9, sequenceid=84, filesize=6.5 K 2024-11-25T03:45:34,916 DEBUG [M:0;d73269e94d72:45215 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f0a1d3ce154d416094cc4dd691877758 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f0a1d3ce154d416094cc4dd691877758 2024-11-25T03:45:34,918 INFO [M:0;d73269e94d72:45215 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f0a1d3ce154d416094cc4dd691877758, entries=2, sequenceid=84, filesize=5.1 K 2024-11-25T03:45:34,919 INFO [M:0;d73269e94d72:45215 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=84, compaction requested=false 2024-11-25T03:45:34,926 INFO [M:0;d73269e94d72:45215 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:45:34,926 DEBUG [M:0;d73269e94d72:45215 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732506334837Disabling compacts and flushes for region at 1732506334837Disabling writes for close at 1732506334837Obtaining lock to block concurrent updates at 1732506334837Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732506334837Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732506334838 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732506334838Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732506334838Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732506334858 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732506334858Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732506334860 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732506334878 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732506334878Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732506334881 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732506334905 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732506334905Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a45bd75: reopening flushed file at 1732506334909 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@603690f6: reopening flushed file at 1732506334912 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8006b81: reopening flushed file at 1732506334915 (+3 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=84, compaction requested=false at 1732506334920 (+5 ms)Writing region close event to WAL at 1732506334926 (+6 ms)Closed at 1732506334926 2024-11-25T03:45:34,930 INFO [M:0;d73269e94d72:45215 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T03:45:34,930 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:45:34,930 INFO [M:0;d73269e94d72:45215 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45215 2024-11-25T03:45:34,931 INFO [M:0;d73269e94d72:45215 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:45:35,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:45:35,040 INFO [M:0;d73269e94d72:45215 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:45:35,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101701895360000, quorum=127.0.0.1:50232, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:45:35,045 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T03:45:35,068 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-25T03:45:35,068 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:35,078 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/d73269e94d72:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732506334659 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: regionserver/d73269e94d72:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732506334386 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/d73269e94d72:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/d73269e94d72:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732506334656 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732506332815 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=391 (was 398), ProcessCount=11 (was 11), AvailableMemoryMB=9243 (was 9678) 2024-11-25T03:45:35,084 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=391, ProcessCount=11, AvailableMemoryMB=9243 2024-11-25T03:45:35,084 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/hadoop-log-dir so I do NOT create it in target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b 2024-11-25T03:45:35,084 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/hadoop-tmp-dir so I do NOT create it in target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b 2024-11-25T03:45:35,084 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:35,087 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/cluster_f59a0090-7256-ea27-ff69-f773868da6b8, deleteOnExit=true 2024-11-25T03:45:35,090 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/cluster_f59a0090-7256-ea27-ff69-f773868da6b8/zookeeper_0, clientPort=64382, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/cluster_f59a0090-7256-ea27-ff69-f773868da6b8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/cluster_f59a0090-7256-ea27-ff69-f773868da6b8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T03:45:35,091 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64382 2024-11-25T03:45:35,106 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1 with version=8 2024-11-25T03:45:35,106 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/hbase-staging 2024-11-25T03:45:35,109 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d73269e94d72:0 server-side Connection retries=45 2024-11-25T03:45:35,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:35,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:35,109 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:45:35,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:35,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:45:35,109 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T03:45:35,110 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:45:35,110 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44319 2024-11-25T03:45:35,112 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44319 connecting to ZooKeeper ensemble=127.0.0.1:64382 2024-11-25T03:45:35,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:443190x0, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:45:35,154 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44319-0x1017018b31c0000 connected 2024-11-25T03:45:35,247 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:35,248 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:35,249 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:45:35,249 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1, hbase.cluster.distributed=false 2024-11-25T03:45:35,251 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:45:35,252 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44319 2024-11-25T03:45:35,253 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44319 2024-11-25T03:45:35,255 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44319 2024-11-25T03:45:35,256 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44319 2024-11-25T03:45:35,256 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44319 2024-11-25T03:45:35,272 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d73269e94d72:0 server-side Connection retries=45 2024-11-25T03:45:35,272 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:35,273 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:35,273 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:45:35,273 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:35,273 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:45:35,273 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T03:45:35,273 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:45:35,274 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44633 2024-11-25T03:45:35,276 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44633 connecting to ZooKeeper ensemble=127.0.0.1:64382 2024-11-25T03:45:35,276 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:35,276 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:35,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:446330x0, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:45:35,290 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44633-0x1017018b31c0001 connected 2024-11-25T03:45:35,290 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:45:35,290 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T03:45:35,291 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/bucket.cache 2024-11-25T03:45:35,291 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T03:45:35,291 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T03:45:35,291 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T03:45:35,294 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T03:45:35,294 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-25T03:45:35,294 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T03:45:35,294 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T03:45:35,294 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T03:45:35,295 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/bucket.persistence is complete. 2024-11-25T03:45:35,295 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T03:45:35,297 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:45:35,299 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44633 2024-11-25T03:45:35,299 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44633 2024-11-25T03:45:35,303 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44633 2024-11-25T03:45:35,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44633 2024-11-25T03:45:35,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44633 2024-11-25T03:45:35,320 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d73269e94d72:0 server-side Connection retries=45 2024-11-25T03:45:35,320 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:35,320 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:35,320 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:45:35,320 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:35,320 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:45:35,320 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T03:45:35,321 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:45:35,321 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37489 2024-11-25T03:45:35,323 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37489 connecting to ZooKeeper ensemble=127.0.0.1:64382 2024-11-25T03:45:35,323 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:35,324 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:35,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:374890x0, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:45:35,332 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:374890x0, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:45:35,332 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37489-0x1017018b31c0002 connected 2024-11-25T03:45:35,333 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T03:45:35,337 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/bucket.cache 2024-11-25T03:45:35,337 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T03:45:35,337 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T03:45:35,337 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T03:45:35,340 INFO [Thread-122 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T03:45:35,340 WARN [Thread-122 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-25T03:45:35,341 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T03:45:35,342 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T03:45:35,342 INFO [Thread-122 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T03:45:35,342 INFO [Thread-122 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/bucket.persistence is complete. 2024-11-25T03:45:35,343 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T03:45:35,346 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:45:35,350 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37489 2024-11-25T03:45:35,351 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37489 2024-11-25T03:45:35,351 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37489 2024-11-25T03:45:35,352 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37489 2024-11-25T03:45:35,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37489 2024-11-25T03:45:35,387 DEBUG [M:0;d73269e94d72:44319 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d73269e94d72:44319 2024-11-25T03:45:35,387 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d73269e94d72,44319,1732506335108 2024-11-25T03:45:35,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:45:35,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:45:35,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:45:35,397 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d73269e94d72,44319,1732506335108 2024-11-25T03:45:35,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:35,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T03:45:35,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:35,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T03:45:35,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:35,406 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T03:45:35,407 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d73269e94d72,44319,1732506335108 from backup master directory 2024-11-25T03:45:35,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:45:35,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d73269e94d72,44319,1732506335108 2024-11-25T03:45:35,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:45:35,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:45:35,416 WARN [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:45:35,416 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d73269e94d72,44319,1732506335108 2024-11-25T03:45:35,417 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/hbase.id] with ID: 95d99bd8-92c5-4f23-914a-65e06eeb727a 2024-11-25T03:45:35,417 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/.tmp/hbase.id 2024-11-25T03:45:35,429 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/hbase.id] 2024-11-25T03:45:35,471 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:35,471 INFO [master/d73269e94d72:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T03:45:35,473 INFO [master/d73269e94d72:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-25T03:45:35,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:35,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:35,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:35,483 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:45:35,483 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T03:45:35,484 INFO [master/d73269e94d72:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:45:35,507 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store 2024-11-25T03:45:35,521 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:45:35,521 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T03:45:35,521 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:45:35,522 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:45:35,522 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T03:45:35,522 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:45:35,522 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:45:35,522 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732506335521Disabling compacts and flushes for region at 1732506335521Disabling writes for close at 1732506335522 (+1 ms)Writing region close event to WAL at 1732506335522Closed at 1732506335522 2024-11-25T03:45:35,522 WARN [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/.initializing 2024-11-25T03:45:35,522 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/WALs/d73269e94d72,44319,1732506335108 2024-11-25T03:45:35,535 INFO [master/d73269e94d72:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:45:35,539 INFO [master/d73269e94d72:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C44319%2C1732506335108, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/WALs/d73269e94d72,44319,1732506335108, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/oldWALs, maxLogs=10 2024-11-25T03:45:35,573 INFO [master/d73269e94d72:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/WALs/d73269e94d72,44319,1732506335108/d73269e94d72%2C44319%2C1732506335108.1732506335540 2024-11-25T03:45:35,573 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T03:45:35,573 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:45:35,574 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:45:35,574 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:35,574 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:35,584 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:35,607 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T03:45:35,607 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:35,607 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:35,608 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:35,625 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T03:45:35,625 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:35,625 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:45:35,626 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:35,630 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T03:45:35,630 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:35,630 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:45:35,630 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:35,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T03:45:35,638 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:35,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:45:35,639 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:35,639 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:35,640 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:35,640 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:35,640 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:35,640 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:45:35,641 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:45:35,661 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:45:35,661 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65748150, jitterRate=-0.02027621865272522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:45:35,663 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732506335575Initializing all the Stores at 1732506335575Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506335575Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506335584 (+9 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506335584Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506335584Cleaning up temporary data from old regions at 1732506335640 (+56 ms)Region opened successfully at 1732506335662 (+22 ms) 2024-11-25T03:45:35,663 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T03:45:35,668 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ce68574, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:45:35,669 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T03:45:35,670 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T03:45:35,670 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T03:45:35,670 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:45:35,670 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T03:45:35,671 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-25T03:45:35,671 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T03:45:35,673 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T03:45:35,674 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T03:45:35,678 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T03:45:35,679 INFO [master/d73269e94d72:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T03:45:35,680 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T03:45:35,689 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T03:45:35,689 INFO [master/d73269e94d72:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T03:45:35,691 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T03:45:35,699 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T03:45:35,701 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T03:45:35,710 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T03:45:35,713 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T03:45:35,721 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T03:45:35,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:45:35,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:45:35,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:45:35,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:35,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:35,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:35,732 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d73269e94d72,44319,1732506335108, sessionid=0x1017018b31c0000, setting cluster-up flag (Was=false) 2024-11-25T03:45:35,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:35,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:35,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:35,784 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T03:45:35,787 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d73269e94d72,44319,1732506335108 2024-11-25T03:45:35,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:35,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:35,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:35,846 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T03:45:35,847 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d73269e94d72,44319,1732506335108 2024-11-25T03:45:35,848 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T03:45:35,850 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:45:35,850 INFO [master/d73269e94d72:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T03:45:35,850 INFO [master/d73269e94d72:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T03:45:35,851 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d73269e94d72,44319,1732506335108 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T03:45:35,853 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:45:35,853 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:45:35,853 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:45:35,853 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:45:35,853 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d73269e94d72:0, corePoolSize=10, maxPoolSize=10 2024-11-25T03:45:35,853 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:35,853 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:45:35,853 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:35,855 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:45:35,855 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T03:45:35,856 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:35,856 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T03:45:35,867 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506365867 2024-11-25T03:45:35,868 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T03:45:35,868 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T03:45:35,868 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T03:45:35,868 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T03:45:35,868 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T03:45:35,868 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T03:45:35,868 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:35,869 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T03:45:35,869 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T03:45:35,869 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T03:45:35,871 INFO [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer(746): ClusterId : 95d99bd8-92c5-4f23-914a-65e06eeb727a 2024-11-25T03:45:35,871 DEBUG [RS:0;d73269e94d72:44633 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T03:45:35,872 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T03:45:35,872 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T03:45:35,877 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.large.0-1732506335872,5,FailOnTimeoutGroup] 2024-11-25T03:45:35,877 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.small.0-1732506335877,5,FailOnTimeoutGroup] 2024-11-25T03:45:35,877 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:35,877 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T03:45:35,877 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:35,877 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:35,880 INFO [RS:1;d73269e94d72:37489 {}] regionserver.HRegionServer(746): ClusterId : 95d99bd8-92c5-4f23-914a-65e06eeb727a 2024-11-25T03:45:35,880 DEBUG [RS:1;d73269e94d72:37489 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T03:45:35,889 DEBUG [RS:0;d73269e94d72:44633 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T03:45:35,889 DEBUG [RS:0;d73269e94d72:44633 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T03:45:35,893 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T03:45:35,893 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1 2024-11-25T03:45:35,901 DEBUG [RS:1;d73269e94d72:37489 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T03:45:35,901 DEBUG [RS:1;d73269e94d72:37489 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T03:45:35,901 DEBUG [RS:0;d73269e94d72:44633 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T03:45:35,901 DEBUG [RS:0;d73269e94d72:44633 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@770f4e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:45:35,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:45:35,906 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T03:45:35,912 DEBUG [RS:1;d73269e94d72:37489 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T03:45:35,913 DEBUG [RS:1;d73269e94d72:37489 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77844d2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:45:35,921 DEBUG [RS:0;d73269e94d72:44633 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d73269e94d72:44633 2024-11-25T03:45:35,922 INFO [RS:0;d73269e94d72:44633 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T03:45:35,922 INFO [RS:0;d73269e94d72:44633 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T03:45:35,922 DEBUG [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T03:45:35,923 INFO [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,44319,1732506335108 with port=44633, startcode=1732506335272 2024-11-25T03:45:35,923 DEBUG [RS:0;d73269e94d72:44633 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T03:45:35,925 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:45:35,925 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:35,925 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:35,926 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T03:45:35,929 DEBUG [RS:1;d73269e94d72:37489 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;d73269e94d72:37489 2024-11-25T03:45:35,930 INFO [RS:1;d73269e94d72:37489 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T03:45:35,930 INFO [RS:1;d73269e94d72:37489 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T03:45:35,930 DEBUG [RS:1;d73269e94d72:37489 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T03:45:35,931 INFO [RS:1;d73269e94d72:37489 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,44319,1732506335108 with port=37489, startcode=1732506335319 2024-11-25T03:45:35,931 DEBUG [RS:1;d73269e94d72:37489 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T03:45:35,933 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45385, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T03:45:35,934 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44319 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d73269e94d72,44633,1732506335272 2024-11-25T03:45:35,934 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44319 {}] master.ServerManager(517): Registering regionserver=d73269e94d72,44633,1732506335272 2024-11-25T03:45:35,942 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:45:35,942 DEBUG [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1 2024-11-25T03:45:35,942 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:35,942 DEBUG [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T03:45:35,942 DEBUG [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T03:45:35,943 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:35,943 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T03:45:35,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:45:35,953 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:45:35,954 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:35,954 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38099, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T03:45:35,954 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:35,954 DEBUG [RS:0;d73269e94d72:44633 {}] zookeeper.ZKUtil(111): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d73269e94d72,44633,1732506335272 2024-11-25T03:45:35,954 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T03:45:35,954 WARN [RS:0;d73269e94d72:44633 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:45:35,954 INFO [RS:0;d73269e94d72:44633 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:45:35,955 DEBUG [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,44633,1732506335272 2024-11-25T03:45:35,955 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44319 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d73269e94d72,37489,1732506335319 2024-11-25T03:45:35,956 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44319 {}] master.ServerManager(517): Registering regionserver=d73269e94d72,37489,1732506335319 2024-11-25T03:45:35,958 DEBUG [RS:1;d73269e94d72:37489 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1 2024-11-25T03:45:35,958 DEBUG [RS:1;d73269e94d72:37489 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T03:45:35,958 DEBUG [RS:1;d73269e94d72:37489 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T03:45:35,969 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:45:35,969 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:35,969 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:35,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:45:35,970 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740 2024-11-25T03:45:35,970 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740 2024-11-25T03:45:35,971 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:45:35,971 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:45:35,971 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:45:35,972 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:45:35,980 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:45:35,981 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64333027, jitterRate=-0.0413631945848465}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:45:35,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732506335904Initializing all the Stores at 1732506335905 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506335905Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506335905Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506335906 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506335906Cleaning up temporary data from old regions at 1732506335971 (+65 ms)Region opened successfully at 1732506335982 (+11 ms) 2024-11-25T03:45:35,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:45:35,982 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:45:35,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:45:35,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:45:35,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:45:35,983 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:45:35,983 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732506335982Disabling compacts and flushes for region at 1732506335982Disabling writes for close at 1732506335982Writing region close event to WAL at 1732506335983 (+1 ms)Closed at 1732506335983 2024-11-25T03:45:35,984 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:45:35,984 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T03:45:35,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T03:45:35,986 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:45:35,987 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T03:45:36,022 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d73269e94d72,37489,1732506335319] 2024-11-25T03:45:36,022 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d73269e94d72,44633,1732506335272] 2024-11-25T03:45:36,022 DEBUG [RS:1;d73269e94d72:37489 {}] zookeeper.ZKUtil(111): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d73269e94d72,37489,1732506335319 2024-11-25T03:45:36,022 WARN [RS:1;d73269e94d72:37489 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:45:36,022 INFO [RS:1;d73269e94d72:37489 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:45:36,023 DEBUG [RS:1;d73269e94d72:37489 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,37489,1732506335319 2024-11-25T03:45:36,023 INFO [RS:0;d73269e94d72:44633 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T03:45:36,025 INFO [RS:0;d73269e94d72:44633 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T03:45:36,026 INFO [RS:0;d73269e94d72:44633 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:45:36,026 INFO [RS:0;d73269e94d72:44633 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,026 INFO [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T03:45:36,026 INFO [RS:0;d73269e94d72:44633 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T03:45:36,027 INFO [RS:0;d73269e94d72:44633 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,027 DEBUG [RS:0;d73269e94d72:44633 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,027 DEBUG [RS:0;d73269e94d72:44633 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,027 DEBUG [RS:0;d73269e94d72:44633 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,027 DEBUG [RS:0;d73269e94d72:44633 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,027 DEBUG [RS:0;d73269e94d72:44633 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,027 DEBUG [RS:0;d73269e94d72:44633 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:45:36,027 DEBUG [RS:0;d73269e94d72:44633 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,028 DEBUG [RS:0;d73269e94d72:44633 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,028 DEBUG [RS:0;d73269e94d72:44633 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,028 DEBUG [RS:0;d73269e94d72:44633 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,028 DEBUG [RS:0;d73269e94d72:44633 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,028 DEBUG [RS:0;d73269e94d72:44633 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,028 DEBUG [RS:0;d73269e94d72:44633 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:45:36,028 DEBUG [RS:0;d73269e94d72:44633 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:45:36,028 INFO [RS:1;d73269e94d72:37489 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T03:45:36,033 INFO [RS:1;d73269e94d72:37489 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T03:45:36,033 INFO [RS:0;d73269e94d72:44633 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,033 INFO [RS:0;d73269e94d72:44633 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,033 INFO [RS:0;d73269e94d72:44633 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,033 INFO [RS:0;d73269e94d72:44633 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,033 INFO [RS:0;d73269e94d72:44633 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,033 INFO [RS:0;d73269e94d72:44633 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,44633,1732506335272-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:45:36,033 INFO [RS:1;d73269e94d72:37489 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:45:36,033 INFO [RS:1;d73269e94d72:37489 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,034 INFO [RS:1;d73269e94d72:37489 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T03:45:36,035 INFO [RS:1;d73269e94d72:37489 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T03:45:36,035 INFO [RS:1;d73269e94d72:37489 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,036 DEBUG [RS:1;d73269e94d72:37489 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,036 DEBUG [RS:1;d73269e94d72:37489 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,036 DEBUG [RS:1;d73269e94d72:37489 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,036 DEBUG [RS:1;d73269e94d72:37489 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,036 DEBUG [RS:1;d73269e94d72:37489 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,036 DEBUG [RS:1;d73269e94d72:37489 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:45:36,037 DEBUG [RS:1;d73269e94d72:37489 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,037 DEBUG [RS:1;d73269e94d72:37489 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,037 DEBUG [RS:1;d73269e94d72:37489 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,037 DEBUG [RS:1;d73269e94d72:37489 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,037 DEBUG [RS:1;d73269e94d72:37489 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,037 DEBUG [RS:1;d73269e94d72:37489 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:36,037 DEBUG [RS:1;d73269e94d72:37489 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:45:36,037 DEBUG [RS:1;d73269e94d72:37489 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:45:36,043 INFO [RS:1;d73269e94d72:37489 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,044 INFO [RS:1;d73269e94d72:37489 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,044 INFO [RS:1;d73269e94d72:37489 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,044 INFO [RS:1;d73269e94d72:37489 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,044 INFO [RS:1;d73269e94d72:37489 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,044 INFO [RS:1;d73269e94d72:37489 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,37489,1732506335319-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:45:36,053 INFO [RS:0;d73269e94d72:44633 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T03:45:36,054 INFO [RS:0;d73269e94d72:44633 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,44633,1732506335272-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,054 INFO [RS:0;d73269e94d72:44633 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,054 INFO [RS:0;d73269e94d72:44633 {}] regionserver.Replication(171): d73269e94d72,44633,1732506335272 started 2024-11-25T03:45:36,066 INFO [RS:1;d73269e94d72:37489 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T03:45:36,067 INFO [RS:1;d73269e94d72:37489 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,37489,1732506335319-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,067 INFO [RS:1;d73269e94d72:37489 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,067 INFO [RS:1;d73269e94d72:37489 {}] regionserver.Replication(171): d73269e94d72,37489,1732506335319 started 2024-11-25T03:45:36,078 INFO [RS:0;d73269e94d72:44633 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,079 INFO [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer(1482): Serving as d73269e94d72,44633,1732506335272, RpcServer on d73269e94d72/172.17.0.2:44633, sessionid=0x1017018b31c0001 2024-11-25T03:45:36,079 DEBUG [RS:0;d73269e94d72:44633 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T03:45:36,079 DEBUG [RS:0;d73269e94d72:44633 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d73269e94d72,44633,1732506335272 2024-11-25T03:45:36,079 DEBUG [RS:0;d73269e94d72:44633 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,44633,1732506335272' 2024-11-25T03:45:36,079 DEBUG [RS:0;d73269e94d72:44633 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T03:45:36,080 DEBUG [RS:0;d73269e94d72:44633 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T03:45:36,080 DEBUG [RS:0;d73269e94d72:44633 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T03:45:36,080 DEBUG [RS:0;d73269e94d72:44633 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T03:45:36,081 DEBUG [RS:0;d73269e94d72:44633 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d73269e94d72,44633,1732506335272 2024-11-25T03:45:36,081 DEBUG [RS:0;d73269e94d72:44633 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,44633,1732506335272' 2024-11-25T03:45:36,081 DEBUG [RS:0;d73269e94d72:44633 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T03:45:36,081 DEBUG [RS:0;d73269e94d72:44633 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T03:45:36,082 DEBUG [RS:0;d73269e94d72:44633 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T03:45:36,082 INFO [RS:0;d73269e94d72:44633 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T03:45:36,082 INFO [RS:0;d73269e94d72:44633 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T03:45:36,087 INFO [RS:1;d73269e94d72:37489 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,087 INFO [RS:1;d73269e94d72:37489 {}] regionserver.HRegionServer(1482): Serving as d73269e94d72,37489,1732506335319, RpcServer on d73269e94d72/172.17.0.2:37489, sessionid=0x1017018b31c0002 2024-11-25T03:45:36,087 DEBUG [RS:1;d73269e94d72:37489 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T03:45:36,087 DEBUG [RS:1;d73269e94d72:37489 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d73269e94d72,37489,1732506335319 2024-11-25T03:45:36,087 DEBUG [RS:1;d73269e94d72:37489 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,37489,1732506335319' 2024-11-25T03:45:36,087 DEBUG [RS:1;d73269e94d72:37489 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T03:45:36,088 DEBUG [RS:1;d73269e94d72:37489 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T03:45:36,089 DEBUG [RS:1;d73269e94d72:37489 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T03:45:36,089 DEBUG [RS:1;d73269e94d72:37489 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T03:45:36,089 DEBUG [RS:1;d73269e94d72:37489 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d73269e94d72,37489,1732506335319 2024-11-25T03:45:36,089 DEBUG [RS:1;d73269e94d72:37489 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,37489,1732506335319' 2024-11-25T03:45:36,089 DEBUG [RS:1;d73269e94d72:37489 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T03:45:36,089 DEBUG [RS:1;d73269e94d72:37489 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T03:45:36,089 DEBUG [RS:1;d73269e94d72:37489 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T03:45:36,089 INFO [RS:1;d73269e94d72:37489 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T03:45:36,090 INFO [RS:1;d73269e94d72:37489 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T03:45:36,138 WARN [d73269e94d72:44319 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:45:36,183 INFO [RS:0;d73269e94d72:44633 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:45:36,184 INFO [RS:0;d73269e94d72:44633 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=d73269e94d72%2C44633%2C1732506335272, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,44633,1732506335272, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/oldWALs, maxLogs=55 2024-11-25T03:45:36,190 INFO [RS:1;d73269e94d72:37489 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:45:36,191 INFO [RS:1;d73269e94d72:37489 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=d73269e94d72%2C37489%2C1732506335319, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,37489,1732506335319, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/oldWALs, maxLogs=55 2024-11-25T03:45:36,205 INFO [RS:0;d73269e94d72:44633 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,44633,1732506335272/d73269e94d72%2C44633%2C1732506335272.1732506336185 2024-11-25T03:45:36,205 DEBUG [RS:0;d73269e94d72:44633 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T03:45:36,213 INFO [RS:1;d73269e94d72:37489 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,37489,1732506335319/d73269e94d72%2C37489%2C1732506335319.1732506336192 2024-11-25T03:45:36,213 DEBUG [RS:1;d73269e94d72:37489 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T03:45:36,388 DEBUG [d73269e94d72:44319 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-25T03:45:36,389 DEBUG [d73269e94d72:44319 {}] balancer.BalancerClusterState(204): Hosts are {d73269e94d72=0} racks are {/default-rack=0} 2024-11-25T03:45:36,389 DEBUG [d73269e94d72:44319 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T03:45:36,390 DEBUG [d73269e94d72:44319 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T03:45:36,390 DEBUG [d73269e94d72:44319 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T03:45:36,390 DEBUG [d73269e94d72:44319 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T03:45:36,390 INFO [d73269e94d72:44319 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T03:45:36,390 INFO [d73269e94d72:44319 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T03:45:36,390 DEBUG [d73269e94d72:44319 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T03:45:36,390 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d73269e94d72,44633,1732506335272 2024-11-25T03:45:36,391 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,44633,1732506335272, state=OPENING 2024-11-25T03:45:36,403 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T03:45:36,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:36,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:36,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:36,422 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:45:36,422 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:36,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,44633,1732506335272}] 2024-11-25T03:45:36,424 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:36,424 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:36,576 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:45:36,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46029, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:45:36,583 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T03:45:36,584 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:45:36,585 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T03:45:36,586 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=d73269e94d72%2C44633%2C1732506335272.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,44633,1732506335272, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/oldWALs, maxLogs=55 2024-11-25T03:45:36,607 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,44633,1732506335272/d73269e94d72%2C44633%2C1732506335272.meta.1732506336587.meta 2024-11-25T03:45:36,608 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T03:45:36,608 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:45:36,608 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T03:45:36,609 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T03:45:36,609 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T03:45:36,609 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T03:45:36,609 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:45:36,609 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T03:45:36,609 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T03:45:36,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T03:45:36,611 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:45:36,611 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:36,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:36,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T03:45:36,612 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:45:36,612 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:36,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:36,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T03:45:36,613 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:45:36,613 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:36,614 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:36,614 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T03:45:36,615 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:45:36,615 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:36,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:36,615 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:45:36,616 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740 2024-11-25T03:45:36,617 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740 2024-11-25T03:45:36,617 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:45:36,617 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:45:36,617 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:45:36,619 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:45:36,619 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70048598, jitterRate=0.043805450201034546}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:45:36,620 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T03:45:36,621 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732506336609Writing region info on filesystem at 1732506336609Initializing all the Stores at 1732506336610 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506336610Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506336610Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506336610Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506336610Cleaning up temporary data from old regions at 1732506336617 (+7 ms)Running coprocessor post-open hooks at 1732506336620 (+3 ms)Region opened successfully at 1732506336620 2024-11-25T03:45:36,621 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732506336576 2024-11-25T03:45:36,625 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T03:45:36,625 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,44633,1732506335272 2024-11-25T03:45:36,625 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T03:45:36,626 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,44633,1732506335272, state=OPEN 2024-11-25T03:45:36,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:45:36,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:45:36,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:45:36,636 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d73269e94d72,44633,1732506335272 2024-11-25T03:45:36,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:36,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:36,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:36,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T03:45:36,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,44633,1732506335272 in 215 msec 2024-11-25T03:45:36,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T03:45:36,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 654 msec 2024-11-25T03:45:36,641 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:45:36,641 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T03:45:36,644 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:45:36,644 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,44633,1732506335272, seqNum=-1] 2024-11-25T03:45:36,644 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:45:36,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34387, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:45:36,651 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 800 msec 2024-11-25T03:45:36,651 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732506336651, completionTime=-1 2024-11-25T03:45:36,651 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-25T03:45:36,651 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:45:36,654 INFO [master/d73269e94d72:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T03:45:36,654 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506396654 2024-11-25T03:45:36,654 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506456654 2024-11-25T03:45:36,654 INFO [master/d73269e94d72:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-25T03:45:36,655 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,44319,1732506335108-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,655 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,44319,1732506335108-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,655 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,44319,1732506335108-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,655 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d73269e94d72:44319, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,655 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,655 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,658 DEBUG [master/d73269e94d72:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T03:45:36,660 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.244sec 2024-11-25T03:45:36,660 INFO [master/d73269e94d72:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T03:45:36,661 INFO [master/d73269e94d72:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T03:45:36,661 INFO [master/d73269e94d72:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T03:45:36,661 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T03:45:36,661 INFO [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T03:45:36,661 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,44319,1732506335108-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:45:36,661 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,44319,1732506335108-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T03:45:36,662 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T03:45:36,662 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T03:45:36,662 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,44319,1732506335108-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:36,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc06048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:45:36,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d73269e94d72,44319,-1 for getting cluster id 2024-11-25T03:45:36,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:45:36,676 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '95d99bd8-92c5-4f23-914a-65e06eeb727a' 2024-11-25T03:45:36,677 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:45:36,677 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "95d99bd8-92c5-4f23-914a-65e06eeb727a" 2024-11-25T03:45:36,677 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e6b9afa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:45:36,677 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d73269e94d72,44319,-1] 2024-11-25T03:45:36,678 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:45:36,678 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:36,680 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:45:36,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:45:36,682 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:45:36,683 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,44633,1732506335272, seqNum=-1] 2024-11-25T03:45:36,684 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:45:36,686 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:45:36,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d73269e94d72,44319,1732506335108 2024-11-25T03:45:36,689 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:45:36,691 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is d73269e94d72,44319,1732506335108 2024-11-25T03:45:36,691 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@19d1ee26 2024-11-25T03:45:36,691 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:45:36,693 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:45:36,694 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44319 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732506336689', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:45:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44319 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732506336689 2024-11-25T03:45:36,696 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732506336689 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:45:36,696 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:36,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44319 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732506336689" procId is: 4 2024-11-25T03:45:36,697 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732506336689 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:45:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:45:36,705 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732506336689-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3b9986d43c04d78f9f6e267e7ae06ad8, NAME => 'testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732506336689', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1 2024-11-25T03:45:36,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732506336689-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:45:36,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732506336689-pool-0 {}] regionserver.HRegion(1722): Closing 3b9986d43c04d78f9f6e267e7ae06ad8, disabling compactions & flushes 2024-11-25T03:45:36,710 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732506336689-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:36,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732506336689-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:36,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732506336689-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. after waiting 0 ms 2024-11-25T03:45:36,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732506336689-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:36,710 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732506336689-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:36,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732506336689-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3b9986d43c04d78f9f6e267e7ae06ad8: Waiting for close lock at 1732506336710Disabling compacts and flushes for region at 1732506336710Disabling writes for close at 1732506336710Writing region close event to WAL at 1732506336710Closed at 1732506336710 2024-11-25T03:45:36,711 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732506336689 execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:45:36,712 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732506336712"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506336712"}]},"ts":"1732506336712"} 2024-11-25T03:45:36,714 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:45:36,715 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732506336689 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:45:36,715 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732506336689","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732506336715"}]},"ts":"1732506336715"} 2024-11-25T03:45:36,717 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732506336689, state=ENABLING in hbase:meta 2024-11-25T03:45:36,717 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {d73269e94d72=0} racks are {/default-rack=0} 2024-11-25T03:45:36,718 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T03:45:36,718 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T03:45:36,718 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T03:45:36,718 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T03:45:36,718 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T03:45:36,718 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T03:45:36,718 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T03:45:36,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732506336689, region=3b9986d43c04d78f9f6e267e7ae06ad8, ASSIGN}] 2024-11-25T03:45:36,719 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732506336689, region=3b9986d43c04d78f9f6e267e7ae06ad8, ASSIGN 2024-11-25T03:45:36,720 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732506336689, region=3b9986d43c04d78f9f6e267e7ae06ad8, ASSIGN; state=OFFLINE, location=d73269e94d72,44633,1732506335272; forceNewPlan=false, retain=false 2024-11-25T03:45:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:45:36,871 INFO [d73269e94d72:44319 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T03:45:36,872 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3b9986d43c04d78f9f6e267e7ae06ad8, regionState=OPENING, regionLocation=d73269e94d72,44633,1732506335272 2024-11-25T03:45:36,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732506336689, region=3b9986d43c04d78f9f6e267e7ae06ad8, ASSIGN because future has completed 2024-11-25T03:45:36,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3b9986d43c04d78f9f6e267e7ae06ad8, server=d73269e94d72,44633,1732506335272}] 2024-11-25T03:45:37,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:45:37,036 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:37,037 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3b9986d43c04d78f9f6e267e7ae06ad8, NAME => 'testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:45:37,037 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732506336689 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:37,037 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:45:37,037 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:37,038 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:37,039 INFO [StoreOpener-3b9986d43c04d78f9f6e267e7ae06ad8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:37,043 INFO [StoreOpener-3b9986d43c04d78f9f6e267e7ae06ad8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b9986d43c04d78f9f6e267e7ae06ad8 columnFamilyName family 2024-11-25T03:45:37,043 DEBUG [StoreOpener-3b9986d43c04d78f9f6e267e7ae06ad8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:37,043 INFO [StoreOpener-3b9986d43c04d78f9f6e267e7ae06ad8-1 {}] regionserver.HStore(327): Store=3b9986d43c04d78f9f6e267e7ae06ad8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:45:37,044 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:37,045 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:37,045 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:37,045 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:37,045 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:37,046 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:37,051 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:45:37,052 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3b9986d43c04d78f9f6e267e7ae06ad8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62305919, jitterRate=-0.07156945765018463}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:45:37,052 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:37,053 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3b9986d43c04d78f9f6e267e7ae06ad8: Running coprocessor pre-open hook at 1732506337038Writing region info on filesystem at 1732506337038Initializing all the Stores at 1732506337038Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506337038Cleaning up temporary data from old regions at 1732506337045 (+7 ms)Running coprocessor post-open hooks at 1732506337052 (+7 ms)Region opened successfully at 1732506337053 (+1 ms) 2024-11-25T03:45:37,053 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8., pid=6, masterSystemTime=1732506337033 2024-11-25T03:45:37,056 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:37,056 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:37,057 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3b9986d43c04d78f9f6e267e7ae06ad8, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,44633,1732506335272 2024-11-25T03:45:37,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3b9986d43c04d78f9f6e267e7ae06ad8, server=d73269e94d72,44633,1732506335272 because future has completed 2024-11-25T03:45:37,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T03:45:37,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3b9986d43c04d78f9f6e267e7ae06ad8, server=d73269e94d72,44633,1732506335272 in 183 msec 2024-11-25T03:45:37,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T03:45:37,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732506336689, region=3b9986d43c04d78f9f6e267e7ae06ad8, ASSIGN in 345 msec 2024-11-25T03:45:37,066 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732506336689 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:45:37,066 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732506336689","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732506337066"}]},"ts":"1732506337066"} 2024-11-25T03:45:37,068 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732506336689, state=ENABLED in hbase:meta 2024-11-25T03:45:37,069 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732506336689 execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:45:37,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732506336689 in 375 msec 2024-11-25T03:45:37,175 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:45:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:45:37,325 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732506336689 completed 2024-11-25T03:45:37,325 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732506336689 get assigned. Timeout = 60000ms 2024-11-25T03:45:37,326 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:45:37,329 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732506336689 assigned to meta. Checking AM states. 2024-11-25T03:45:37,330 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:45:37,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732506336689 assigned. 2024-11-25T03:45:37,334 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732506336689', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8., hostname=d73269e94d72,44633,1732506335272, seqNum=2] 2024-11-25T03:45:37,339 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3b9986d43c04d78f9f6e267e7ae06ad8 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-25T03:45:37,349 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e0b4905edf664ba48fc60c4094f3aaef 2024-11-25T03:45:37,361 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/.tmp/family/e0b4905edf664ba48fc60c4094f3aaef is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-25T03:45:37,361 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/.tmp/family/e0b4905edf664ba48fc60c4094f3aaef 2024-11-25T03:45:37,364 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/.tmp/family/e0b4905edf664ba48fc60c4094f3aaef as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/family/e0b4905edf664ba48fc60c4094f3aaef 2024-11-25T03:45:37,365 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/family/e0b4905edf664ba48fc60c4094f3aaef, delay=916 ms 2024-11-25T03:45:37,367 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/family/e0b4905edf664ba48fc60c4094f3aaef, entries=2, sequenceid=6, filesize=4.9 K 2024-11-25T03:45:37,368 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 3b9986d43c04d78f9f6e267e7ae06ad8 in 29ms, sequenceid=6, compaction requested=false 2024-11-25T03:45:37,368 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732506336689' 2024-11-25T03:45:37,369 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3b9986d43c04d78f9f6e267e7ae06ad8: 2024-11-25T03:45:38,166 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T03:45:38,166 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T03:45:38,169 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:45:38,169 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T03:45:38,170 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T03:45:38,170 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T03:45:38,171 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732506336689 2024-11-25T03:45:38,171 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732506336689 Metrics about Tables on a single HBase RegionServer 2024-11-25T03:45:38,282 DEBUG [hfile-prefetch-1732506334386 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key e0b4905edf664ba48fc60c4094f3aaef_0. 2024-11-25T03:45:38,283 DEBUG [hfile-prefetch-1732506334386 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key e0b4905edf664ba48fc60c4094f3aaef_117. 2024-11-25T03:45:38,284 DEBUG [hfile-prefetch-1732506334386 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/family/e0b4905edf664ba48fc60c4094f3aaef, with total blocks 2, and data blocks 1 2024-11-25T03:45:38,284 DEBUG [hfile-prefetch-1732506334386 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-25T03:45:38,284 DEBUG [hfile-prefetch-1732506334386 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/family/e0b4905edf664ba48fc60c4094f3aaef, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-25T03:45:38,284 DEBUG [hfile-prefetch-1732506334386 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/family/e0b4905edf664ba48fc60c4094f3aaef. Will sleep 100ms and try the verification again. 2024-11-25T03:45:38,369 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;d73269e94d72:44633,5,FailOnTimeoutGroup] 2024-11-25T03:45:38,369 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd73269e94d72,44633,1732506335272' ***** 2024-11-25T03:45:38,369 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-25T03:45:38,370 INFO [RS:0;d73269e94d72:44633 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T03:45:38,384 DEBUG [hfile-prefetch-1732506334386 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/family/e0b4905edf664ba48fc60c4094f3aaef, with total blocks 2, and data blocks 1 2024-11-25T03:45:38,384 DEBUG [hfile-prefetch-1732506334386 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-25T03:45:38,384 DEBUG [hfile-prefetch-1732506334386 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/family/e0b4905edf664ba48fc60c4094f3aaef, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-25T03:45:38,384 DEBUG [hfile-prefetch-1732506334386 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/family/e0b4905edf664ba48fc60c4094f3aaef. Will sleep 100ms and try the verification again. 2024-11-25T03:45:38,420 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T03:45:38,420 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T03:45:38,420 INFO [RS:0;d73269e94d72:44633 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/bucket.persistence 2024-11-25T03:45:38,420 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T03:45:38,420 INFO [RS:0;d73269e94d72:44633 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T03:45:38,420 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T03:45:38,430 DEBUG [RS:0;d73269e94d72:44633 {}] bucket.BucketCache(1487): Thread RS:0;d73269e94d72:44633 started persisting bucket cache to file 2024-11-25T03:45:38,430 DEBUG [RS:0;d73269e94d72:44633 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T03:45:38,430 DEBUG [RS:0;d73269e94d72:44633 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T03:45:38,435 DEBUG [RS:0;d73269e94d72:44633 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-25T03:45:38,435 DEBUG [RS:0;d73269e94d72:44633 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732506338430 2024-11-25T03:45:38,435 DEBUG [RS:0;d73269e94d72:44633 {}] bucket.BucketCache(1509): Thread RS:0;d73269e94d72:44633 finished persisting bucket cache to file, renaming 2024-11-25T03:45:38,435 INFO [RS:0;d73269e94d72:44633 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T03:45:38,435 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T03:45:38,435 INFO [RS:0;d73269e94d72:44633 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T03:45:38,435 INFO [RS:0;d73269e94d72:44633 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T03:45:38,435 INFO [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer(3091): Received CLOSE for 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:38,436 INFO [regionserver/d73269e94d72:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:45:38,436 INFO [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer(959): stopping server d73269e94d72,44633,1732506335272 2024-11-25T03:45:38,436 INFO [RS:0;d73269e94d72:44633 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:45:38,436 INFO [RS:0;d73269e94d72:44633 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d73269e94d72:44633. 2024-11-25T03:45:38,436 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T03:45:38,436 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3b9986d43c04d78f9f6e267e7ae06ad8, disabling compactions & flushes 2024-11-25T03:45:38,436 DEBUG [RS:0;d73269e94d72:44633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:45:38,436 DEBUG [RS:0;d73269e94d72:44633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:38,436 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:38,436 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:38,436 INFO [RS:0;d73269e94d72:44633 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T03:45:38,436 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. after waiting 0 ms 2024-11-25T03:45:38,436 INFO [RS:0;d73269e94d72:44633 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T03:45:38,437 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:38,437 INFO [RS:0;d73269e94d72:44633 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T03:45:38,437 INFO [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T03:45:38,437 INFO [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-25T03:45:38,437 DEBUG [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer(1325): Online Regions={3b9986d43c04d78f9f6e267e7ae06ad8=testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8., 1588230740=hbase:meta,,1.1588230740} 2024-11-25T03:45:38,437 DEBUG [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:38,437 DEBUG [StoreFileCloser-3b9986d43c04d78f9f6e267e7ae06ad8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/family/e0b4905edf664ba48fc60c4094f3aaef 2024-11-25T03:45:38,437 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T03:45:38,437 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:45:38,437 WARN [hfile-prefetch-1732506334386 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/family/e0b4905edf664ba48fc60c4094f3aaef, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 10 more 2024-11-25T03:45:38,438 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:45:38,438 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:45:38,438 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:45:38,438 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:45:38,438 DEBUG [hfile-prefetch-1732506334386 {}] hfile.PrefetchExecutor(121): Prefetch completed for e0b4905edf664ba48fc60c4094f3aaef 2024-11-25T03:45:38,438 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-25T03:45:38,440 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T03:45:38,440 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T03:45:38,440 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T03:45:38,440 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T03:45:38,444 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T03:45:38,444 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732506338440 2024-11-25T03:45:38,444 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T03:45:38,444 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-25T03:45:38,444 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:38,444 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3b9986d43c04d78f9f6e267e7ae06ad8: Waiting for close lock at 1732506338436Running coprocessor pre-close hooks at 1732506338436Disabling compacts and flushes for region at 1732506338436Disabling writes for close at 1732506338436Writing region close event to WAL at 1732506338438 (+2 ms)Running coprocessor post-close hooks at 1732506338444 (+6 ms)Closed at 1732506338444 2024-11-25T03:45:38,445 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:38,452 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 584cfcee3dcb4d7fb6dbfc157152fa16 2024-11-25T03:45:38,464 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/info/584cfcee3dcb4d7fb6dbfc157152fa16 is 209, key is testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8./info:regioninfo/1732506337057/Put/seqid=0 2024-11-25T03:45:38,465 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/info/584cfcee3dcb4d7fb6dbfc157152fa16 2024-11-25T03:45:38,473 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 940fea4f4f704789a366d802c67d38a1 2024-11-25T03:45:38,489 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/ns/940fea4f4f704789a366d802c67d38a1 is 43, key is default/ns:d/1732506336646/Put/seqid=0 2024-11-25T03:45:38,490 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/ns/940fea4f4f704789a366d802c67d38a1 2024-11-25T03:45:38,499 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 843f64276a914ee09b60871a9fe0208e 2024-11-25T03:45:38,513 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/table/843f64276a914ee09b60871a9fe0208e is 80, key is testBlockEvictionOnGracefulStop1732506336689/table:state/1732506337066/Put/seqid=0 2024-11-25T03:45:38,513 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/table/843f64276a914ee09b60871a9fe0208e 2024-11-25T03:45:38,515 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/info/584cfcee3dcb4d7fb6dbfc157152fa16 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/info/584cfcee3dcb4d7fb6dbfc157152fa16 2024-11-25T03:45:38,517 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/info/584cfcee3dcb4d7fb6dbfc157152fa16, delay=930 ms 2024-11-25T03:45:38,518 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/info/584cfcee3dcb4d7fb6dbfc157152fa16, entries=10, sequenceid=11, filesize=7.0 K 2024-11-25T03:45:38,518 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/ns/940fea4f4f704789a366d802c67d38a1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/ns/940fea4f4f704789a366d802c67d38a1 2024-11-25T03:45:38,519 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/ns/940fea4f4f704789a366d802c67d38a1, delay=910 ms 2024-11-25T03:45:38,520 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/ns/940fea4f4f704789a366d802c67d38a1, entries=2, sequenceid=11, filesize=5.0 K 2024-11-25T03:45:38,520 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/table/843f64276a914ee09b60871a9fe0208e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/table/843f64276a914ee09b60871a9fe0208e 2024-11-25T03:45:38,521 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/table/843f64276a914ee09b60871a9fe0208e, delay=977 ms 2024-11-25T03:45:38,522 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/table/843f64276a914ee09b60871a9fe0208e, entries=2, sequenceid=11, filesize=5.3 K 2024-11-25T03:45:38,523 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 85ms, sequenceid=11, compaction requested=false 2024-11-25T03:45:38,524 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/info/584cfcee3dcb4d7fb6dbfc157152fa16 2024-11-25T03:45:38,524 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/ns/940fea4f4f704789a366d802c67d38a1 2024-11-25T03:45:38,525 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/table/843f64276a914ee09b60871a9fe0208e 2024-11-25T03:45:38,530 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-25T03:45:38,530 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:45:38,531 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:45:38,531 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732506338437Running coprocessor pre-close hooks at 1732506338437Disabling compacts and flushes for region at 1732506338437Disabling writes for close at 1732506338438 (+1 ms)Obtaining lock to block concurrent updates at 1732506338438Preparing flush snapshotting stores in 1588230740 at 1732506338438Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732506338439 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732506338439Flushing 1588230740/info: creating writer at 1732506338439Flushing 1588230740/info: appending metadata at 1732506338463 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732506338463Flushing 1588230740/ns: creating writer at 1732506338467 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732506338488 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732506338488Flushing 1588230740/table: creating writer at 1732506338492 (+4 ms)Flushing 1588230740/table: appending metadata at 1732506338512 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732506338512Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9089aee: reopening flushed file at 1732506338515 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77a04716: reopening flushed file at 1732506338518 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@468844a1: reopening flushed file at 1732506338520 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 85ms, sequenceid=11, compaction requested=false at 1732506338523 (+3 ms)Writing region close event to WAL at 1732506338525 (+2 ms)Running coprocessor post-close hooks at 1732506338530 (+5 ms)Closed at 1732506338531 (+1 ms) 2024-11-25T03:45:38,531 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T03:45:38,637 INFO [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer(976): stopping server d73269e94d72,44633,1732506335272; all regions closed. 2024-11-25T03:45:38,640 DEBUG [RS:0;d73269e94d72:44633 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/oldWALs 2024-11-25T03:45:38,640 INFO [RS:0;d73269e94d72:44633 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d73269e94d72%2C44633%2C1732506335272.meta:.meta(num 1732506336587) 2024-11-25T03:45:38,642 DEBUG [RS:0;d73269e94d72:44633 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/oldWALs 2024-11-25T03:45:38,642 INFO [RS:0;d73269e94d72:44633 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d73269e94d72%2C44633%2C1732506335272:(num 1732506336185) 2024-11-25T03:45:38,642 DEBUG [RS:0;d73269e94d72:44633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:38,643 INFO [RS:0;d73269e94d72:44633 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:45:38,643 INFO [RS:0;d73269e94d72:44633 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:45:38,643 INFO [RS:0;d73269e94d72:44633 {}] hbase.ChoreService(370): Chore service for: regionserver/d73269e94d72:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T03:45:38,643 INFO [RS:0;d73269e94d72:44633 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:45:38,643 INFO [regionserver/d73269e94d72:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:45:38,643 INFO [RS:0;d73269e94d72:44633 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44633 2024-11-25T03:45:38,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:45:38,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d73269e94d72,44633,1732506335272 2024-11-25T03:45:38,658 INFO [RS:0;d73269e94d72:44633 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:45:38,659 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d73269e94d72,44633,1732506335272] 2024-11-25T03:45:38,678 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d73269e94d72,44633,1732506335272 already deleted, retry=false 2024-11-25T03:45:38,679 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of d73269e94d72,44633,1732506335272 on d73269e94d72,44319,1732506335108 2024-11-25T03:45:38,684 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure d73269e94d72,44633,1732506335272, splitWal=true, meta=true 2024-11-25T03:45:38,687 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for d73269e94d72,44633,1732506335272 (carryingMeta=true) d73269e94d72,44633,1732506335272/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4813cb66[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-25T03:45:38,687 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure d73269e94d72,44633,1732506335272, splitWal=true, meta=true 2024-11-25T03:45:38,688 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure d73269e94d72,44633,1732506335272, splitWal=true, meta=true, isMeta: true 2024-11-25T03:45:38,689 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,44633,1732506335272-splitting 2024-11-25T03:45:38,689 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,44633,1732506335272-splitting dir is empty, no logs to split. 2024-11-25T03:45:38,690 INFO [PEWorker-1 {}] master.SplitWALManager(105): d73269e94d72,44633,1732506335272 WAL count=0, meta=true 2024-11-25T03:45:38,693 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,44633,1732506335272-splitting dir is empty, no logs to split. 2024-11-25T03:45:38,693 INFO [PEWorker-1 {}] master.SplitWALManager(105): d73269e94d72,44633,1732506335272 WAL count=0, meta=true 2024-11-25T03:45:38,693 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if d73269e94d72,44633,1732506335272 WAL splitting is done? wals=0, meta=true 2024-11-25T03:45:38,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T03:45:38,698 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:45:38,699 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T03:45:38,768 INFO [RS:0;d73269e94d72:44633 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:45:38,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:45:38,768 INFO [RS:0;d73269e94d72:44633 {}] regionserver.HRegionServer(1031): Exiting; stopping=d73269e94d72,44633,1732506335272; zookeeper connection closed. 2024-11-25T03:45:38,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44633-0x1017018b31c0001, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:45:38,768 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@211a9527 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@211a9527 2024-11-25T03:45:38,849 DEBUG [d73269e94d72:44319 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-25T03:45:38,850 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d73269e94d72,37489,1732506335319 2024-11-25T03:45:38,851 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,37489,1732506335319, state=OPENING 2024-11-25T03:45:38,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:45:38,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:45:38,858 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:45:38,858 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:38,858 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:38,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,37489,1732506335319}] 2024-11-25T03:45:38,887 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d73269e94d72:0 server-side Connection retries=45 2024-11-25T03:45:38,887 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:38,888 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:38,888 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:45:38,888 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:45:38,888 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:45:38,888 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T03:45:38,888 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:45:38,889 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33721 2024-11-25T03:45:38,891 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33721 connecting to ZooKeeper ensemble=127.0.0.1:64382 2024-11-25T03:45:38,891 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:38,891 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:38,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:337210x0, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:45:38,901 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33721-0x1017018b31c0003 connected 2024-11-25T03:45:38,901 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:33721-0x1017018b31c0003, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-25T03:45:38,901 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-25T03:45:38,902 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T03:45:38,903 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/bucket.cache 2024-11-25T03:45:38,903 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T03:45:38,904 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T03:45:38,904 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T03:45:38,904 INFO [Thread-209 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T03:45:38,904 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T03:45:38,905 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T03:45:38,905 INFO [Thread-209 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-25T03:45:38,906 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:33721-0x1017018b31c0003, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T03:45:38,906 INFO [Thread-209 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-25T03:45:38,908 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33721-0x1017018b31c0003, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:45:38,920 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33721 2024-11-25T03:45:38,920 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33721 2024-11-25T03:45:38,924 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33721 2024-11-25T03:45:38,925 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33721 2024-11-25T03:45:38,928 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33721 2024-11-25T03:45:38,932 DEBUG [Thread-209 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-25T03:45:38,933 INFO [Thread-209 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T03:45:38,933 INFO [Thread-209 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-25T03:45:38,933 INFO [Thread-209 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/bucket.persistence is complete. 2024-11-25T03:45:38,933 INFO [RS:2;d73269e94d72:33721 {}] regionserver.HRegionServer(746): ClusterId : 95d99bd8-92c5-4f23-914a-65e06eeb727a 2024-11-25T03:45:38,933 DEBUG [RS:2;d73269e94d72:33721 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T03:45:38,943 DEBUG [RS:2;d73269e94d72:33721 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T03:45:38,943 DEBUG [RS:2;d73269e94d72:33721 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T03:45:38,953 DEBUG [RS:2;d73269e94d72:33721 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T03:45:38,954 DEBUG [RS:2;d73269e94d72:33721 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5397a5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:45:38,973 DEBUG [RS:2;d73269e94d72:33721 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;d73269e94d72:33721 2024-11-25T03:45:38,973 INFO [RS:2;d73269e94d72:33721 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T03:45:38,974 INFO [RS:2;d73269e94d72:33721 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T03:45:38,974 DEBUG [RS:2;d73269e94d72:33721 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T03:45:38,976 INFO [RS:2;d73269e94d72:33721 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,44319,1732506335108 with port=33721, startcode=1732506338887 2024-11-25T03:45:38,976 DEBUG [RS:2;d73269e94d72:33721 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T03:45:38,980 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44443, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T03:45:38,981 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44319 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d73269e94d72,33721,1732506338887 2024-11-25T03:45:38,981 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44319 {}] master.ServerManager(517): Registering regionserver=d73269e94d72,33721,1732506338887 2024-11-25T03:45:38,983 DEBUG [RS:2;d73269e94d72:33721 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1 2024-11-25T03:45:38,983 DEBUG [RS:2;d73269e94d72:33721 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T03:45:38,983 DEBUG [RS:2;d73269e94d72:33721 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T03:45:39,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:45:39,004 DEBUG [RS:2;d73269e94d72:33721 {}] zookeeper.ZKUtil(111): regionserver:33721-0x1017018b31c0003, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d73269e94d72,33721,1732506338887 2024-11-25T03:45:39,004 WARN [RS:2;d73269e94d72:33721 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:45:39,004 INFO [RS:2;d73269e94d72:33721 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:45:39,004 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d73269e94d72,33721,1732506338887] 2024-11-25T03:45:39,004 DEBUG [RS:2;d73269e94d72:33721 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,33721,1732506338887 2024-11-25T03:45:39,011 INFO [RS:2;d73269e94d72:33721 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T03:45:39,011 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:45:39,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47189, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:45:39,014 INFO [RS:2;d73269e94d72:33721 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T03:45:39,015 INFO [RS:2;d73269e94d72:33721 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:45:39,015 INFO [RS:2;d73269e94d72:33721 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:39,016 INFO [RS:2;d73269e94d72:33721 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T03:45:39,017 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T03:45:39,017 INFO [RS:2;d73269e94d72:33721 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T03:45:39,017 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:45:39,017 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T03:45:39,018 INFO [RS:2;d73269e94d72:33721 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:39,018 DEBUG [RS:2;d73269e94d72:33721 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:39,018 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=d73269e94d72%2C37489%2C1732506335319.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,37489,1732506335319, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/oldWALs, maxLogs=55 2024-11-25T03:45:39,018 DEBUG [RS:2;d73269e94d72:33721 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:39,018 DEBUG [RS:2;d73269e94d72:33721 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:39,018 DEBUG [RS:2;d73269e94d72:33721 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:39,019 DEBUG [RS:2;d73269e94d72:33721 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:39,019 DEBUG [RS:2;d73269e94d72:33721 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:45:39,019 DEBUG [RS:2;d73269e94d72:33721 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:39,019 DEBUG [RS:2;d73269e94d72:33721 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:39,019 DEBUG [RS:2;d73269e94d72:33721 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:39,019 DEBUG [RS:2;d73269e94d72:33721 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:39,019 DEBUG [RS:2;d73269e94d72:33721 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:39,019 DEBUG [RS:2;d73269e94d72:33721 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:45:39,019 DEBUG [RS:2;d73269e94d72:33721 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:45:39,019 DEBUG [RS:2;d73269e94d72:33721 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:45:39,026 INFO [RS:2;d73269e94d72:33721 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:39,026 INFO [RS:2;d73269e94d72:33721 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:39,026 INFO [RS:2;d73269e94d72:33721 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:39,027 INFO [RS:2;d73269e94d72:33721 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:39,027 INFO [RS:2;d73269e94d72:33721 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:39,027 INFO [RS:2;d73269e94d72:33721 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,33721,1732506338887-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:45:39,041 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,37489,1732506335319/d73269e94d72%2C37489%2C1732506335319.meta.1732506339019.meta 2024-11-25T03:45:39,041 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T03:45:39,041 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:45:39,042 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T03:45:39,042 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T03:45:39,042 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T03:45:39,042 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T03:45:39,042 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:45:39,043 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T03:45:39,043 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T03:45:39,043 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T03:45:39,044 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:45:39,044 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:39,046 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/info/584cfcee3dcb4d7fb6dbfc157152fa16, delay=980 ms 2024-11-25T03:45:39,047 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/info/584cfcee3dcb4d7fb6dbfc157152fa16 2024-11-25T03:45:39,047 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:39,047 INFO [RS:2;d73269e94d72:33721 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T03:45:39,047 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T03:45:39,047 INFO [RS:2;d73269e94d72:33721 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,33721,1732506338887-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:39,048 INFO [RS:2;d73269e94d72:33721 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:39,048 INFO [RS:2;d73269e94d72:33721 {}] regionserver.Replication(171): d73269e94d72,33721,1732506338887 started 2024-11-25T03:45:39,048 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:45:39,048 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:39,050 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/ns/940fea4f4f704789a366d802c67d38a1, delay=945 ms 2024-11-25T03:45:39,051 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/ns/940fea4f4f704789a366d802c67d38a1 2024-11-25T03:45:39,051 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:39,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T03:45:39,052 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:45:39,052 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:39,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:39,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T03:45:39,053 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:45:39,053 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:39,055 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/table/843f64276a914ee09b60871a9fe0208e, delay=986 ms 2024-11-25T03:45:39,060 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/table/843f64276a914ee09b60871a9fe0208e 2024-11-25T03:45:39,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:45:39,060 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:45:39,061 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740 2024-11-25T03:45:39,061 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740 2024-11-25T03:45:39,061 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:45:39,061 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:45:39,062 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:45:39,063 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:45:39,064 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67523202, jitterRate=0.00617411732673645}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:45:39,064 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T03:45:39,065 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732506339043Writing region info on filesystem at 1732506339043Initializing all the Stores at 1732506339043Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506339043Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506339043Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506339043Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506339043Cleaning up temporary data from old regions at 1732506339062 (+19 ms)Running coprocessor post-open hooks at 1732506339064 (+2 ms)Region opened successfully at 1732506339065 (+1 ms) 2024-11-25T03:45:39,068 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732506339011 2024-11-25T03:45:39,069 INFO [RS:2;d73269e94d72:33721 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:45:39,070 INFO [RS:2;d73269e94d72:33721 {}] regionserver.HRegionServer(1482): Serving as d73269e94d72,33721,1732506338887, RpcServer on d73269e94d72/172.17.0.2:33721, sessionid=0x1017018b31c0003 2024-11-25T03:45:39,070 DEBUG [RS:2;d73269e94d72:33721 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T03:45:39,070 DEBUG [RS:2;d73269e94d72:33721 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d73269e94d72,33721,1732506338887 2024-11-25T03:45:39,070 DEBUG [RS:2;d73269e94d72:33721 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,33721,1732506338887' 2024-11-25T03:45:39,070 DEBUG [RS:2;d73269e94d72:33721 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T03:45:39,071 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T03:45:39,071 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T03:45:39,071 DEBUG [RS:2;d73269e94d72:33721 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T03:45:39,072 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=d73269e94d72,37489,1732506335319 2024-11-25T03:45:39,073 DEBUG [RS:2;d73269e94d72:33721 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T03:45:39,073 DEBUG [RS:2;d73269e94d72:33721 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T03:45:39,073 DEBUG [RS:2;d73269e94d72:33721 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d73269e94d72,33721,1732506338887 2024-11-25T03:45:39,073 DEBUG [RS:2;d73269e94d72:33721 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,33721,1732506338887' 2024-11-25T03:45:39,073 DEBUG [RS:2;d73269e94d72:33721 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T03:45:39,073 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,37489,1732506335319, state=OPEN 2024-11-25T03:45:39,073 DEBUG [RS:2;d73269e94d72:33721 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T03:45:39,089 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44319 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=d73269e94d72,37489,1732506335319, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T03:45:39,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33721-0x1017018b31c0003, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:45:39,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:45:39,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:45:39,144 DEBUG [RS:2;d73269e94d72:33721 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T03:45:39,144 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d73269e94d72,37489,1732506335319 2024-11-25T03:45:39,144 INFO [RS:2;d73269e94d72:33721 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T03:45:39,144 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:39,144 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:39,144 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:45:39,144 INFO [RS:2;d73269e94d72:33721 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T03:45:39,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-25T03:45:39,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,37489,1732506335319 in 286 msec 2024-11-25T03:45:39,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T03:45:39,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 450 msec 2024-11-25T03:45:39,148 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): d73269e94d72,44633,1732506335272 had 2 regions 2024-11-25T03:45:39,149 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure d73269e94d72,44633,1732506335272, splitWal=true, meta=true, isMeta: false 2024-11-25T03:45:39,150 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,44633,1732506335272-splitting dir is empty, no logs to split. 2024-11-25T03:45:39,150 INFO [PEWorker-3 {}] master.SplitWALManager(105): d73269e94d72,44633,1732506335272 WAL count=0, meta=false 2024-11-25T03:45:39,151 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,44633,1732506335272-splitting dir is empty, no logs to split. 2024-11-25T03:45:39,151 INFO [PEWorker-3 {}] master.SplitWALManager(105): d73269e94d72,44633,1732506335272 WAL count=0, meta=false 2024-11-25T03:45:39,151 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if d73269e94d72,44633,1732506335272 WAL splitting is done? wals=0, meta=false 2024-11-25T03:45:39,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732506336689, region=3b9986d43c04d78f9f6e267e7ae06ad8, ASSIGN}] 2024-11-25T03:45:39,154 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732506336689, region=3b9986d43c04d78f9f6e267e7ae06ad8, ASSIGN 2024-11-25T03:45:39,154 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732506336689, region=3b9986d43c04d78f9f6e267e7ae06ad8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T03:45:39,245 INFO [RS:2;d73269e94d72:33721 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:45:39,246 INFO [RS:2;d73269e94d72:33721 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=d73269e94d72%2C33721%2C1732506338887, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,33721,1732506338887, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/oldWALs, maxLogs=55 2024-11-25T03:45:39,262 INFO [RS:2;d73269e94d72:33721 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/WALs/d73269e94d72,33721,1732506338887/d73269e94d72%2C33721%2C1732506338887.1732506339246 2024-11-25T03:45:39,264 DEBUG [RS:2;d73269e94d72:33721 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T03:45:39,305 DEBUG [d73269e94d72:44319 {}] balancer.BalancerClusterState(204): Hosts are {d73269e94d72=0} racks are {/default-rack=0} 2024-11-25T03:45:39,305 DEBUG [d73269e94d72:44319 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T03:45:39,305 DEBUG [d73269e94d72:44319 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T03:45:39,305 DEBUG [d73269e94d72:44319 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T03:45:39,305 DEBUG [d73269e94d72:44319 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T03:45:39,305 INFO [d73269e94d72:44319 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T03:45:39,305 INFO [d73269e94d72:44319 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T03:45:39,305 DEBUG [d73269e94d72:44319 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T03:45:39,306 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3b9986d43c04d78f9f6e267e7ae06ad8, regionState=OPENING, regionLocation=d73269e94d72,33721,1732506338887 2024-11-25T03:45:39,307 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server d73269e94d72:44633 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d73269e94d72/172.17.0.2:44633 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:45:39,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d73269e94d72,44633,1732506335272, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d73269e94d72,44633,1732506335272, seqNum=-1, error=java.net.ConnectException: Call to address=d73269e94d72:44633 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d73269e94d72/172.17.0.2:44633 2024-11-25T03:45:39,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=d73269e94d72,44633,1732506335272, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-25T03:45:39,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=d73269e94d72,44633,1732506335272, seqNum=-1 from cache 2024-11-25T03:45:39,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address d73269e94d72:44633 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d73269e94d72/172.17.0.2:44633 2024-11-25T03:45:39,413 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:45:39,414 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,37489,1732506335319, seqNum=-1] 2024-11-25T03:45:39,414 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:45:39,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45571, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:45:39,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732506336689, region=3b9986d43c04d78f9f6e267e7ae06ad8, ASSIGN because future has completed 2024-11-25T03:45:39,420 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3b9986d43c04d78f9f6e267e7ae06ad8, server=d73269e94d72,33721,1732506338887}] 2024-11-25T03:45:39,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T03:45:39,570 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:45:39,571 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:45:39,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:39,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:39,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:45:39,571 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T03:45:39,571 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1925617390, stopped=false 2024-11-25T03:45:39,571 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d73269e94d72,44319,1732506335108 2024-11-25T03:45:39,573 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:45:39,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59873, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:45:39,577 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:39,577 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 3b9986d43c04d78f9f6e267e7ae06ad8, NAME => 'testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:45:39,578 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732506336689 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:39,578 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:45:39,578 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:39,578 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:39,579 INFO [StoreOpener-3b9986d43c04d78f9f6e267e7ae06ad8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:39,580 INFO [StoreOpener-3b9986d43c04d78f9f6e267e7ae06ad8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b9986d43c04d78f9f6e267e7ae06ad8 columnFamilyName family 2024-11-25T03:45:39,581 DEBUG [StoreOpener-3b9986d43c04d78f9f6e267e7ae06ad8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:45:39,583 DEBUG [StoreFileOpener-3b9986d43c04d78f9f6e267e7ae06ad8-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/family/e0b4905edf664ba48fc60c4094f3aaef, delay=960 ms 2024-11-25T03:45:39,585 DEBUG [StoreOpener-3b9986d43c04d78f9f6e267e7ae06ad8-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/family/e0b4905edf664ba48fc60c4094f3aaef 2024-11-25T03:45:39,585 INFO [StoreOpener-3b9986d43c04d78f9f6e267e7ae06ad8-1 {}] regionserver.HStore(327): Store=3b9986d43c04d78f9f6e267e7ae06ad8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:45:39,585 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:39,586 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:39,587 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:39,587 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:39,587 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:39,589 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:39,590 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 3b9986d43c04d78f9f6e267e7ae06ad8; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62281310, jitterRate=-0.07193616032600403}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:45:39,590 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:39,591 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 3b9986d43c04d78f9f6e267e7ae06ad8: Running coprocessor pre-open hook at 1732506339578Writing region info on filesystem at 1732506339578Initializing all the Stores at 1732506339579 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506339579Cleaning up temporary data from old regions at 1732506339587 (+8 ms)Running coprocessor post-open hooks at 1732506339590 (+3 ms)Region opened successfully at 1732506339591 (+1 ms) 2024-11-25T03:45:39,591 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8., pid=11, masterSystemTime=1732506339572 2024-11-25T03:45:39,594 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:39,594 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:39,595 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3b9986d43c04d78f9f6e267e7ae06ad8, regionState=OPEN, openSeqNum=10, regionLocation=d73269e94d72,33721,1732506338887 2024-11-25T03:45:39,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3b9986d43c04d78f9f6e267e7ae06ad8, server=d73269e94d72,33721,1732506338887 because future has completed 2024-11-25T03:45:39,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-25T03:45:39,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 3b9986d43c04d78f9f6e267e7ae06ad8, server=d73269e94d72,33721,1732506338887 in 179 msec 2024-11-25T03:45:39,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-25T03:45:39,603 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server d73269e94d72,44633,1732506335272 after splitting done 2024-11-25T03:45:39,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732506336689, region=3b9986d43c04d78f9f6e267e7ae06ad8, ASSIGN in 449 msec 2024-11-25T03:45:39,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure d73269e94d72,44633,1732506335272, splitWal=true, meta=true in 923 msec 2024-11-25T03:45:39,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33721-0x1017018b31c0003, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:45:39,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:45:39,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:45:39,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33721-0x1017018b31c0003, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:39,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:39,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:39,711 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:45:39,712 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:45:39,713 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:45:39,713 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:45:39,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:39,714 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33721-0x1017018b31c0003, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:45:39,714 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:45:39,714 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd73269e94d72,37489,1732506335319' ***** 2024-11-25T03:45:39,714 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T03:45:39,715 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd73269e94d72,33721,1732506338887' ***** 2024-11-25T03:45:39,715 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T03:45:39,715 INFO [RS:1;d73269e94d72:37489 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T03:45:39,715 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T03:45:39,715 INFO [RS:1;d73269e94d72:37489 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/bucket.persistence 2024-11-25T03:45:39,715 INFO [RS:2;d73269e94d72:33721 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T03:45:39,715 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T03:45:39,716 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T03:45:39,716 INFO [RS:1;d73269e94d72:37489 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T03:45:39,716 INFO [RS:2;d73269e94d72:33721 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b1717af-2618-ae34-d84e-04c685d70a7b/bucket.persistence 2024-11-25T03:45:39,716 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T03:45:39,716 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T03:45:39,716 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T03:45:39,716 INFO [RS:2;d73269e94d72:33721 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T03:45:39,716 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T03:45:39,716 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T03:45:39,726 DEBUG [RS:1;d73269e94d72:37489 {}] bucket.BucketCache(1487): Thread RS:1;d73269e94d72:37489 started persisting bucket cache to file 2024-11-25T03:45:39,726 DEBUG [RS:1;d73269e94d72:37489 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T03:45:39,726 DEBUG [RS:1;d73269e94d72:37489 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T03:45:39,726 DEBUG [RS:2;d73269e94d72:33721 {}] bucket.BucketCache(1487): Thread RS:2;d73269e94d72:33721 started persisting bucket cache to file 2024-11-25T03:45:39,726 DEBUG [RS:2;d73269e94d72:33721 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T03:45:39,726 DEBUG [RS:2;d73269e94d72:33721 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T03:45:39,727 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T03:45:39,728 INFO [regionserver/d73269e94d72:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:45:39,728 DEBUG [RS:2;d73269e94d72:33721 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T03:45:39,728 DEBUG [RS:1;d73269e94d72:37489 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T03:45:39,729 DEBUG [RS:1;d73269e94d72:37489 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732506339726 2024-11-25T03:45:39,729 DEBUG [RS:2;d73269e94d72:33721 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732506339726 2024-11-25T03:45:39,729 DEBUG [RS:2;d73269e94d72:33721 {}] bucket.BucketCache(1509): Thread RS:2;d73269e94d72:33721 finished persisting bucket cache to file, renaming 2024-11-25T03:45:39,729 INFO [RS:2;d73269e94d72:33721 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T03:45:39,729 INFO [RS:2;d73269e94d72:33721 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T03:45:39,729 INFO [RS:2;d73269e94d72:33721 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T03:45:39,729 INFO [RS:2;d73269e94d72:33721 {}] regionserver.HRegionServer(3091): Received CLOSE for 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:39,730 INFO [RS:2;d73269e94d72:33721 {}] regionserver.HRegionServer(959): stopping server d73269e94d72,33721,1732506338887 2024-11-25T03:45:39,730 INFO [RS:2;d73269e94d72:33721 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:45:39,730 INFO [RS:2;d73269e94d72:33721 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;d73269e94d72:33721. 2024-11-25T03:45:39,730 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T03:45:39,730 DEBUG [RS:2;d73269e94d72:33721 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:45:39,730 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3b9986d43c04d78f9f6e267e7ae06ad8, disabling compactions & flushes 2024-11-25T03:45:39,730 DEBUG [RS:2;d73269e94d72:33721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:39,730 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:39,730 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:39,730 INFO [RS:2;d73269e94d72:33721 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T03:45:39,730 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. after waiting 0 ms 2024-11-25T03:45:39,730 DEBUG [RS:2;d73269e94d72:33721 {}] regionserver.HRegionServer(1325): Online Regions={3b9986d43c04d78f9f6e267e7ae06ad8=testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8.} 2024-11-25T03:45:39,730 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:39,730 DEBUG [RS:2;d73269e94d72:33721 {}] regionserver.HRegionServer(1351): Waiting on 3b9986d43c04d78f9f6e267e7ae06ad8 2024-11-25T03:45:39,731 DEBUG [StoreFileCloser-3b9986d43c04d78f9f6e267e7ae06ad8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/family/e0b4905edf664ba48fc60c4094f3aaef 2024-11-25T03:45:39,736 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/default/testBlockEvictionOnGracefulStop1732506336689/3b9986d43c04d78f9f6e267e7ae06ad8/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-25T03:45:39,736 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:39,737 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3b9986d43c04d78f9f6e267e7ae06ad8: Waiting for close lock at 1732506339730Running coprocessor pre-close hooks at 1732506339730Disabling compacts and flushes for region at 1732506339730Disabling writes for close at 1732506339730Writing region close event to WAL at 1732506339731 (+1 ms)Running coprocessor post-close hooks at 1732506339736 (+5 ms)Closed at 1732506339736 2024-11-25T03:45:39,737 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8. 2024-11-25T03:45:39,744 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T03:45:39,747 DEBUG [RS:1;d73269e94d72:37489 {}] bucket.BucketCache(1509): Thread RS:1;d73269e94d72:37489 finished persisting bucket cache to file, renaming 2024-11-25T03:45:39,747 WARN [RS:1;d73269e94d72:37489 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-25T03:45:39,747 INFO [RS:1;d73269e94d72:37489 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T03:45:39,747 INFO [RS:1;d73269e94d72:37489 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T03:45:39,747 INFO [RS:1;d73269e94d72:37489 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T03:45:39,747 INFO [RS:1;d73269e94d72:37489 {}] regionserver.HRegionServer(959): stopping server d73269e94d72,37489,1732506335319 2024-11-25T03:45:39,747 INFO [RS:1;d73269e94d72:37489 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:45:39,748 INFO [RS:1;d73269e94d72:37489 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;d73269e94d72:37489. 2024-11-25T03:45:39,748 DEBUG [RS:1;d73269e94d72:37489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:45:39,748 DEBUG [RS:1;d73269e94d72:37489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:39,748 INFO [RS:1;d73269e94d72:37489 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T03:45:39,748 INFO [RS:1;d73269e94d72:37489 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T03:45:39,748 INFO [RS:1;d73269e94d72:37489 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T03:45:39,748 INFO [RS:1;d73269e94d72:37489 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T03:45:39,748 INFO [RS:1;d73269e94d72:37489 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T03:45:39,748 DEBUG [RS:1;d73269e94d72:37489 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-25T03:45:39,749 DEBUG [RS:1;d73269e94d72:37489 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:45:39,749 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T03:45:39,749 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:45:39,749 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:45:39,749 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:45:39,749 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:45:39,749 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:45:39,749 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-25T03:45:39,759 INFO [regionserver/d73269e94d72:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:45:39,763 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 58ceb4bfaab0405e94f8432fedc97090 2024-11-25T03:45:39,779 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/info/58ceb4bfaab0405e94f8432fedc97090 is 209, key is testBlockEvictionOnGracefulStop1732506336689,,1732506336694.3b9986d43c04d78f9f6e267e7ae06ad8./info:regioninfo/1732506339595/Put/seqid=0 2024-11-25T03:45:39,779 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/info/58ceb4bfaab0405e94f8432fedc97090 2024-11-25T03:45:39,781 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/.tmp/info/58ceb4bfaab0405e94f8432fedc97090 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/info/58ceb4bfaab0405e94f8432fedc97090 2024-11-25T03:45:39,782 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/info/58ceb4bfaab0405e94f8432fedc97090, delay=927 ms 2024-11-25T03:45:39,783 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/info/58ceb4bfaab0405e94f8432fedc97090, entries=8, sequenceid=19, filesize=6.6 K 2024-11-25T03:45:39,784 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 35ms, sequenceid=19, compaction requested=false 2024-11-25T03:45:39,785 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/info/584cfcee3dcb4d7fb6dbfc157152fa16 2024-11-25T03:45:39,785 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/info/58ceb4bfaab0405e94f8432fedc97090 2024-11-25T03:45:39,785 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/ns/940fea4f4f704789a366d802c67d38a1 2024-11-25T03:45:39,786 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/table/843f64276a914ee09b60871a9fe0208e 2024-11-25T03:45:39,791 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-25T03:45:39,791 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:45:39,791 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:45:39,791 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732506339749Running coprocessor pre-close hooks at 1732506339749Disabling compacts and flushes for region at 1732506339749Disabling writes for close at 1732506339749Obtaining lock to block concurrent updates at 1732506339749Preparing flush snapshotting stores in 1588230740 at 1732506339749Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732506339750 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732506339750Flushing 1588230740/info: creating writer at 1732506339750Flushing 1588230740/info: appending metadata at 1732506339778 (+28 ms)Flushing 1588230740/info: closing flushed file at 1732506339778Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@631b6e06: reopening flushed file at 1732506339781 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 35ms, sequenceid=19, compaction requested=false at 1732506339784 (+3 ms)Writing region close event to WAL at 1732506339787 (+3 ms)Running coprocessor post-close hooks at 1732506339791 (+4 ms)Closed at 1732506339791 2024-11-25T03:45:39,791 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T03:45:39,931 INFO [RS:2;d73269e94d72:33721 {}] regionserver.HRegionServer(976): stopping server d73269e94d72,33721,1732506338887; all regions closed. 2024-11-25T03:45:39,934 DEBUG [RS:2;d73269e94d72:33721 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/oldWALs 2024-11-25T03:45:39,935 INFO [RS:2;d73269e94d72:33721 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d73269e94d72%2C33721%2C1732506338887:(num 1732506339246) 2024-11-25T03:45:39,935 DEBUG [RS:2;d73269e94d72:33721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:39,935 INFO [RS:2;d73269e94d72:33721 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:45:39,935 INFO [RS:2;d73269e94d72:33721 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:45:39,936 INFO [RS:2;d73269e94d72:33721 {}] hbase.ChoreService(370): Chore service for: regionserver/d73269e94d72:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T03:45:39,936 INFO [RS:2;d73269e94d72:33721 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T03:45:39,936 INFO [RS:2;d73269e94d72:33721 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T03:45:39,936 INFO [regionserver/d73269e94d72:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:45:39,936 INFO [RS:2;d73269e94d72:33721 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T03:45:39,937 INFO [RS:2;d73269e94d72:33721 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:45:39,937 INFO [RS:2;d73269e94d72:33721 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33721 2024-11-25T03:45:39,949 INFO [RS:1;d73269e94d72:37489 {}] regionserver.HRegionServer(976): stopping server d73269e94d72,37489,1732506335319; all regions closed. 2024-11-25T03:45:39,951 DEBUG [RS:1;d73269e94d72:37489 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/oldWALs 2024-11-25T03:45:39,951 INFO [RS:1;d73269e94d72:37489 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d73269e94d72%2C37489%2C1732506335319.meta:.meta(num 1732506339019) 2024-11-25T03:45:39,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:45:39,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33721-0x1017018b31c0003, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d73269e94d72,33721,1732506338887 2024-11-25T03:45:39,952 INFO [RS:2;d73269e94d72:33721 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:45:39,953 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d73269e94d72,33721,1732506338887] 2024-11-25T03:45:39,954 DEBUG [RS:1;d73269e94d72:37489 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/oldWALs 2024-11-25T03:45:39,954 INFO [RS:1;d73269e94d72:37489 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d73269e94d72%2C37489%2C1732506335319:(num 1732506336192) 2024-11-25T03:45:39,954 DEBUG [RS:1;d73269e94d72:37489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:45:39,954 INFO [RS:1;d73269e94d72:37489 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:45:39,954 INFO [RS:1;d73269e94d72:37489 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:45:39,955 INFO [RS:1;d73269e94d72:37489 {}] hbase.ChoreService(370): Chore service for: regionserver/d73269e94d72:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T03:45:39,955 INFO [RS:1;d73269e94d72:37489 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:45:39,955 INFO [regionserver/d73269e94d72:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:45:39,955 INFO [RS:1;d73269e94d72:37489 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37489 2024-11-25T03:45:39,973 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d73269e94d72,33721,1732506338887 already deleted, retry=false 2024-11-25T03:45:39,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:45:39,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d73269e94d72,37489,1732506335319 2024-11-25T03:45:39,973 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d73269e94d72,33721,1732506338887 expired; onlineServers=1 2024-11-25T03:45:39,973 INFO [RS:1;d73269e94d72:37489 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:45:39,984 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d73269e94d72,37489,1732506335319] 2024-11-25T03:45:39,994 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d73269e94d72,37489,1732506335319 already deleted, retry=false 2024-11-25T03:45:39,994 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d73269e94d72,37489,1732506335319 expired; onlineServers=0 2024-11-25T03:45:39,994 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd73269e94d72,44319,1732506335108' ***** 2024-11-25T03:45:39,994 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T03:45:39,995 INFO [M:0;d73269e94d72:44319 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:45:39,995 INFO [M:0;d73269e94d72:44319 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:45:39,995 DEBUG [M:0;d73269e94d72:44319 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T03:45:39,995 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T03:45:39,995 DEBUG [M:0;d73269e94d72:44319 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T03:45:39,995 DEBUG [master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.large.0-1732506335872 {}] cleaner.HFileCleaner(306): Exit Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.large.0-1732506335872,5,FailOnTimeoutGroup] 2024-11-25T03:45:39,995 DEBUG [master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.small.0-1732506335877 {}] cleaner.HFileCleaner(306): Exit Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.small.0-1732506335877,5,FailOnTimeoutGroup] 2024-11-25T03:45:39,996 INFO [M:0;d73269e94d72:44319 {}] hbase.ChoreService(370): Chore service for: master/d73269e94d72:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T03:45:39,996 INFO [M:0;d73269e94d72:44319 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:45:39,996 DEBUG [M:0;d73269e94d72:44319 {}] master.HMaster(1795): Stopping service threads 2024-11-25T03:45:39,996 INFO [M:0;d73269e94d72:44319 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T03:45:39,996 INFO [M:0;d73269e94d72:44319 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:45:39,997 INFO [M:0;d73269e94d72:44319 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:45:39,997 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T03:45:40,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T03:45:40,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:45:40,005 DEBUG [M:0;d73269e94d72:44319 {}] zookeeper.ZKUtil(347): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T03:45:40,005 WARN [M:0;d73269e94d72:44319 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T03:45:40,005 INFO [M:0;d73269e94d72:44319 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/.lastflushedseqids 2024-11-25T03:45:40,011 INFO [M:0;d73269e94d72:44319 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:45:40,011 INFO [M:0;d73269e94d72:44319 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T03:45:40,011 DEBUG [M:0;d73269e94d72:44319 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T03:45:40,012 INFO [M:0;d73269e94d72:44319 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:45:40,012 DEBUG [M:0;d73269e94d72:44319 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:45:40,012 DEBUG [M:0;d73269e94d72:44319 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T03:45:40,012 DEBUG [M:0;d73269e94d72:44319 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:45:40,012 INFO [M:0;d73269e94d72:44319 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.51 KB heapSize=49.91 KB 2024-11-25T03:45:40,022 DEBUG [M:0;d73269e94d72:44319 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b1fc34529d544981894ebd7a8f671cd5 2024-11-25T03:45:40,036 DEBUG [M:0;d73269e94d72:44319 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1fc34529d544981894ebd7a8f671cd5 is 82, key is hbase:meta,,1/info:regioninfo/1732506339071/Put/seqid=0 2024-11-25T03:45:40,037 INFO [M:0;d73269e94d72:44319 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1fc34529d544981894ebd7a8f671cd5 2024-11-25T03:45:40,044 DEBUG [M:0;d73269e94d72:44319 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ec0d2ad6e3be4b429976afe84069a514 2024-11-25T03:45:40,055 DEBUG [M:0;d73269e94d72:44319 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ec0d2ad6e3be4b429976afe84069a514 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732506337070/Put/seqid=0 2024-11-25T03:45:40,056 INFO [M:0;d73269e94d72:44319 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.28 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ec0d2ad6e3be4b429976afe84069a514 2024-11-25T03:45:40,062 DEBUG [M:0;d73269e94d72:44319 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 836c4e887596448f89ef80764d76f653 2024-11-25T03:45:40,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33721-0x1017018b31c0003, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:45:40,063 INFO [RS:2;d73269e94d72:33721 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:45:40,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33721-0x1017018b31c0003, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:45:40,063 INFO [RS:2;d73269e94d72:33721 {}] regionserver.HRegionServer(1031): Exiting; stopping=d73269e94d72,33721,1732506338887; zookeeper connection closed. 2024-11-25T03:45:40,063 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5bff60b4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5bff60b4 2024-11-25T03:45:40,075 DEBUG [M:0;d73269e94d72:44319 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/836c4e887596448f89ef80764d76f653 is 69, key is d73269e94d72,33721,1732506338887/rs:state/1732506338981/Put/seqid=0 2024-11-25T03:45:40,075 INFO [M:0;d73269e94d72:44319 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/836c4e887596448f89ef80764d76f653 2024-11-25T03:45:40,077 INFO [M:0;d73269e94d72:44319 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 836c4e887596448f89ef80764d76f653 2024-11-25T03:45:40,077 DEBUG [M:0;d73269e94d72:44319 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1fc34529d544981894ebd7a8f671cd5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b1fc34529d544981894ebd7a8f671cd5 2024-11-25T03:45:40,079 INFO [M:0;d73269e94d72:44319 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b1fc34529d544981894ebd7a8f671cd5, entries=14, sequenceid=102, filesize=5.9 K 2024-11-25T03:45:40,079 DEBUG [M:0;d73269e94d72:44319 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ec0d2ad6e3be4b429976afe84069a514 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ec0d2ad6e3be4b429976afe84069a514 2024-11-25T03:45:40,081 INFO [M:0;d73269e94d72:44319 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ec0d2ad6e3be4b429976afe84069a514, entries=11, sequenceid=102, filesize=6.5 K 2024-11-25T03:45:40,081 DEBUG [M:0;d73269e94d72:44319 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/836c4e887596448f89ef80764d76f653 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/836c4e887596448f89ef80764d76f653 2024-11-25T03:45:40,082 INFO [M:0;d73269e94d72:44319 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 836c4e887596448f89ef80764d76f653 2024-11-25T03:45:40,083 INFO [M:0;d73269e94d72:44319 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/48f63306-7bbc-b225-8603-e709b16f2d1f/9c346678-e470-a9e0-b016-5bd191c9a9a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/836c4e887596448f89ef80764d76f653, entries=3, sequenceid=102, filesize=5.3 K 2024-11-25T03:45:40,083 INFO [M:0;d73269e94d72:44319 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=102, compaction requested=false 2024-11-25T03:45:40,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:45:40,084 INFO [RS:1;d73269e94d72:37489 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:45:40,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37489-0x1017018b31c0002, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:45:40,084 INFO [RS:1;d73269e94d72:37489 {}] regionserver.HRegionServer(1031): Exiting; stopping=d73269e94d72,37489,1732506335319; zookeeper connection closed. 2024-11-25T03:45:40,084 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@356abd19 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@356abd19 2024-11-25T03:45:40,084 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-25T03:45:40,088 INFO [M:0;d73269e94d72:44319 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:45:40,088 DEBUG [M:0;d73269e94d72:44319 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732506340011Disabling compacts and flushes for region at 1732506340011Disabling writes for close at 1732506340012 (+1 ms)Obtaining lock to block concurrent updates at 1732506340012Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732506340012Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40460, getHeapSize=51048, getOffHeapSize=0, getCellsCount=127 at 1732506340013 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732506340013Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732506340014 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732506340036 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732506340036Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732506340038 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732506340055 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732506340055Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732506340057 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732506340074 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732506340074Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76c6db08: reopening flushed file at 1732506340077 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21e6d2f0: reopening flushed file at 1732506340079 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b21669f: reopening flushed file at 1732506340081 (+2 ms)Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=102, compaction requested=false at 1732506340083 (+2 ms)Writing region close event to WAL at 1732506340088 (+5 ms)Closed at 1732506340088 2024-11-25T03:45:40,089 INFO [M:0;d73269e94d72:44319 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T03:45:40,089 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:45:40,089 INFO [M:0;d73269e94d72:44319 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44319 2024-11-25T03:45:40,089 INFO [M:0;d73269e94d72:44319 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:45:40,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:45:40,200 INFO [M:0;d73269e94d72:44319 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:45:40,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44319-0x1017018b31c0000, quorum=127.0.0.1:64382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:45:40,206 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T03:45:40,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-25T03:45:40,209 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:45:40,216 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 391), ProcessCount=11 (was 11), AvailableMemoryMB=10501 (was 9243) - AvailableMemoryMB LEAK? -