2024-11-25 02:54:43,873 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-25 02:54:43,891 main DEBUG Took 0.015902 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 02:54:43,892 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 02:54:43,892 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 02:54:43,893 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 02:54:43,894 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 02:54:43,901 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 02:54:43,913 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 02:54:43,915 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 02:54:43,915 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 02:54:43,916 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 02:54:43,916 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 02:54:43,917 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 02:54:43,918 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 02:54:43,918 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 02:54:43,918 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 02:54:43,919 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 02:54:43,920 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 02:54:43,920 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 02:54:43,920 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 02:54:43,921 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 02:54:43,921 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 02:54:43,921 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 02:54:43,922 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 02:54:43,922 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 02:54:43,923 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 02:54:43,923 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 02:54:43,923 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 02:54:43,924 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 02:54:43,924 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 02:54:43,924 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 02:54:43,925 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 02:54:43,925 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 02:54:43,926 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 02:54:43,928 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 02:54:43,929 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 02:54:43,930 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 02:54:43,931 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 02:54:43,931 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 02:54:43,939 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 02:54:43,941 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 02:54:43,943 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 02:54:43,943 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 02:54:43,944 main DEBUG createAppenders(={Console}) 2024-11-25 02:54:43,945 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-25 02:54:43,945 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-25 02:54:43,946 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-25 02:54:43,947 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 02:54:43,947 main DEBUG OutputStream closed 2024-11-25 02:54:43,947 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 02:54:43,947 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 02:54:43,948 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-25 02:54:44,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 02:54:44,040 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 02:54:44,041 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 02:54:44,042 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 02:54:44,043 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 02:54:44,044 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 02:54:44,044 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 02:54:44,045 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 02:54:44,045 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 02:54:44,046 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 02:54:44,046 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 02:54:44,047 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 02:54:44,047 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 02:54:44,048 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 02:54:44,048 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 02:54:44,049 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 02:54:44,049 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 02:54:44,050 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 02:54:44,052 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 02:54:44,053 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-25 02:54:44,053 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 02:54:44,054 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-25T02:54:44,067 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-25 02:54:44,071 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 02:54:44,071 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T02:54:44,314 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f 2024-11-25T02:54:44,339 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=12501 2024-11-25T02:54:44,438 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T02:54:44,527 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:44,542 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/cluster_cd6985d1-feb9-a12e-422e-dc6af6b486ba, deleteOnExit=true 2024-11-25T02:54:44,672 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/cluster_cd6985d1-feb9-a12e-422e-dc6af6b486ba/zookeeper_0, clientPort=63189, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/cluster_cd6985d1-feb9-a12e-422e-dc6af6b486ba/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/cluster_cd6985d1-feb9-a12e-422e-dc6af6b486ba/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T02:54:44,684 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63189 2024-11-25T02:54:44,685 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:44,779 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a with version=8 2024-11-25T02:54:44,780 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/hbase-staging 2024-11-25T02:54:44,874 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T02:54:45,104 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2c5ea3e41165:0 server-side Connection retries=45 2024-11-25T02:54:45,113 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:45,115 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:45,127 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T02:54:45,128 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:45,128 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T02:54:45,272 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T02:54:45,353 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T02:54:45,391 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T02:54:45,402 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T02:54:45,406 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T02:54:45,441 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 94448 (auto-detected) 2024-11-25T02:54:45,442 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-25T02:54:45,463 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37837 2024-11-25T02:54:45,486 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37837 connecting to ZooKeeper ensemble=127.0.0.1:63189 2024-11-25T02:54:45,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:378370x0, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T02:54:45,594 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37837-0x1017133be700000 connected 2024-11-25T02:54:45,686 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:45,688 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:45,699 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T02:54:45,703 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a, hbase.cluster.distributed=false 2024-11-25T02:54:45,730 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T02:54:45,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37837 2024-11-25T02:54:45,737 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37837 2024-11-25T02:54:45,737 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37837 2024-11-25T02:54:45,738 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37837 2024-11-25T02:54:45,738 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37837 2024-11-25T02:54:45,816 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2c5ea3e41165:0 server-side Connection retries=45 2024-11-25T02:54:45,817 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:45,818 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:45,818 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T02:54:45,818 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:45,818 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T02:54:45,821 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T02:54:45,824 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T02:54:45,825 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34269 2024-11-25T02:54:45,827 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34269 connecting to ZooKeeper ensemble=127.0.0.1:63189 2024-11-25T02:54:45,828 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:45,829 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:45,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:342690x0, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T02:54:45,849 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:342690x0, quorum=127.0.0.1:63189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T02:54:45,849 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34269-0x1017133be700001 connected 2024-11-25T02:54:45,857 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T02:54:45,874 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/bucket.cache 2024-11-25T02:54:45,882 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T02:54:45,884 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T02:54:45,887 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T02:54:45,889 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T02:54:45,889 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-25T02:54:45,897 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T02:54:45,897 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T02:54:45,898 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/bucket.persistence is complete. 2024-11-25T02:54:45,905 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T02:54:45,908 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T02:54:45,913 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T02:54:45,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34269 2024-11-25T02:54:45,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34269 2024-11-25T02:54:45,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34269 2024-11-25T02:54:45,918 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34269 2024-11-25T02:54:45,921 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34269 2024-11-25T02:54:45,944 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2c5ea3e41165:0 server-side Connection retries=45 2024-11-25T02:54:45,944 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:45,944 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:45,944 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T02:54:45,945 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:45,945 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T02:54:45,945 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T02:54:45,945 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T02:54:45,946 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35111 2024-11-25T02:54:45,948 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35111 connecting to ZooKeeper ensemble=127.0.0.1:63189 2024-11-25T02:54:45,949 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:45,950 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:45,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:351110x0, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T02:54:45,976 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:351110x0, quorum=127.0.0.1:63189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T02:54:45,976 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35111-0x1017133be700002 connected 2024-11-25T02:54:45,976 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T02:54:45,977 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/bucket.cache 2024-11-25T02:54:45,977 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T02:54:45,978 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T02:54:45,978 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T02:54:45,978 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T02:54:45,979 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-25T02:54:45,979 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T02:54:45,979 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T02:54:45,980 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T02:54:45,980 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/bucket.persistence is complete. 2024-11-25T02:54:45,981 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T02:54:45,984 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T02:54:45,986 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35111 2024-11-25T02:54:45,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35111 2024-11-25T02:54:45,990 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35111 2024-11-25T02:54:45,991 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35111 2024-11-25T02:54:45,992 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35111 2024-11-25T02:54:46,014 DEBUG [M:0;2c5ea3e41165:37837 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2c5ea3e41165:37837 2024-11-25T02:54:46,015 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2c5ea3e41165,37837,1732503284943 2024-11-25T02:54:46,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T02:54:46,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T02:54:46,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T02:54:46,034 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2c5ea3e41165,37837,1732503284943 2024-11-25T02:54:46,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T02:54:46,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T02:54:46,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:46,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:46,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:46,068 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T02:54:46,069 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2c5ea3e41165,37837,1732503284943 from backup master directory 2024-11-25T02:54:46,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2c5ea3e41165,37837,1732503284943 2024-11-25T02:54:46,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T02:54:46,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T02:54:46,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T02:54:46,082 WARN [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T02:54:46,083 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2c5ea3e41165,37837,1732503284943 2024-11-25T02:54:46,085 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T02:54:46,086 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T02:54:46,094 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/hbase.id] with ID: 41bb3781-782d-416c-a0f1-7020b486cc6d 2024-11-25T02:54:46,095 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/.tmp/hbase.id 2024-11-25T02:54:46,100 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/hbase.id] 2024-11-25T02:54:46,135 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:46,143 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T02:54:46,168 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-25T02:54:46,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:46,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:46,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:46,216 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T02:54:46,220 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T02:54:46,262 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T02:54:46,268 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T02:54:46,316 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store 2024-11-25T02:54:46,339 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T02:54:46,343 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T02:54:46,345 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T02:54:46,346 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T02:54:46,346 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T02:54:46,348 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T02:54:46,349 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T02:54:46,349 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T02:54:46,350 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732503286345Disabling compacts and flushes for region at 1732503286345Disabling writes for close at 1732503286349 (+4 ms)Writing region close event to WAL at 1732503286349Closed at 1732503286349 2024-11-25T02:54:46,351 WARN [master/2c5ea3e41165:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/.initializing 2024-11-25T02:54:46,351 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/WALs/2c5ea3e41165,37837,1732503284943 2024-11-25T02:54:46,365 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T02:54:46,381 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2c5ea3e41165%2C37837%2C1732503284943, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/WALs/2c5ea3e41165,37837,1732503284943, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/oldWALs, maxLogs=10 2024-11-25T02:54:46,424 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/WALs/2c5ea3e41165,37837,1732503284943/2c5ea3e41165%2C37837%2C1732503284943.1732503286387 2024-11-25T02:54:46,425 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T02:54:46,425 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T02:54:46,426 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T02:54:46,429 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:46,430 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:46,477 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:46,519 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T02:54:46,525 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:46,528 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:46,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:46,535 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T02:54:46,535 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:46,536 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T02:54:46,536 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:46,541 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T02:54:46,542 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:46,542 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T02:54:46,543 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:46,547 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T02:54:46,548 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:46,548 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T02:54:46,548 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:46,552 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:46,552 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:46,555 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:46,556 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:46,558 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T02:54:46,560 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:46,567 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T02:54:46,568 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66689315, jitterRate=-0.00625176727771759}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T02:54:46,576 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732503286442Initializing all the Stores at 1732503286442Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732503286443 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732503286444 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732503286444Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732503286444Cleaning up temporary data from old regions at 1732503286556 (+112 ms)Region opened successfully at 1732503286576 (+20 ms) 2024-11-25T02:54:46,578 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T02:54:46,608 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fcccfc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2c5ea3e41165/172.17.0.2:0 2024-11-25T02:54:46,639 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T02:54:46,652 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T02:54:46,652 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T02:54:46,655 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T02:54:46,656 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T02:54:46,662 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-25T02:54:46,663 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T02:54:46,687 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T02:54:46,696 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T02:54:46,748 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T02:54:46,750 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T02:54:46,752 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T02:54:46,764 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T02:54:46,767 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T02:54:46,771 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T02:54:46,781 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T02:54:46,783 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T02:54:46,791 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T02:54:46,810 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T02:54:46,816 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T02:54:46,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T02:54:46,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T02:54:46,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:46,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T02:54:46,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:46,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:46,839 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2c5ea3e41165,37837,1732503284943, sessionid=0x1017133be700000, setting cluster-up flag (Was=false) 2024-11-25T02:54:46,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:46,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:46,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:46,891 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T02:54:46,894 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2c5ea3e41165,37837,1732503284943 2024-11-25T02:54:46,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:46,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:46,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:46,941 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T02:54:46,943 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2c5ea3e41165,37837,1732503284943 2024-11-25T02:54:46,951 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T02:54:46,997 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.HRegionServer(746): ClusterId : 41bb3781-782d-416c-a0f1-7020b486cc6d 2024-11-25T02:54:46,997 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer(746): ClusterId : 41bb3781-782d-416c-a0f1-7020b486cc6d 2024-11-25T02:54:47,000 DEBUG [RS:1;2c5ea3e41165:35111 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T02:54:47,000 DEBUG [RS:0;2c5ea3e41165:34269 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T02:54:47,028 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T02:54:47,035 DEBUG [RS:1;2c5ea3e41165:35111 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T02:54:47,035 DEBUG [RS:0;2c5ea3e41165:34269 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T02:54:47,035 DEBUG [RS:0;2c5ea3e41165:34269 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T02:54:47,035 DEBUG [RS:1;2c5ea3e41165:35111 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T02:54:47,043 DEBUG [RS:0;2c5ea3e41165:34269 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T02:54:47,043 DEBUG [RS:1;2c5ea3e41165:35111 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T02:54:47,043 DEBUG [RS:0;2c5ea3e41165:34269 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f410e38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2c5ea3e41165/172.17.0.2:0 2024-11-25T02:54:47,043 DEBUG [RS:1;2c5ea3e41165:35111 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109bf842, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2c5ea3e41165/172.17.0.2:0 2024-11-25T02:54:47,043 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T02:54:47,053 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T02:54:47,058 DEBUG [RS:1;2c5ea3e41165:35111 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2c5ea3e41165:35111 2024-11-25T02:54:47,061 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T02:54:47,061 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T02:54:47,061 DEBUG [RS:1;2c5ea3e41165:35111 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T02:54:47,063 DEBUG [RS:0;2c5ea3e41165:34269 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2c5ea3e41165:34269 2024-11-25T02:54:47,064 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T02:54:47,064 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T02:54:47,064 DEBUG [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T02:54:47,064 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.HRegionServer(2659): reportForDuty to master=2c5ea3e41165,37837,1732503284943 with port=35111, startcode=1732503285943 2024-11-25T02:54:47,061 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2c5ea3e41165,37837,1732503284943 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T02:54:47,065 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer(2659): reportForDuty to master=2c5ea3e41165,37837,1732503284943 with port=34269, startcode=1732503285784 2024-11-25T02:54:47,071 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2c5ea3e41165:0, corePoolSize=5, maxPoolSize=5 2024-11-25T02:54:47,071 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2c5ea3e41165:0, corePoolSize=5, maxPoolSize=5 2024-11-25T02:54:47,072 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2c5ea3e41165:0, corePoolSize=5, maxPoolSize=5 2024-11-25T02:54:47,072 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2c5ea3e41165:0, corePoolSize=5, maxPoolSize=5 2024-11-25T02:54:47,072 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2c5ea3e41165:0, corePoolSize=10, maxPoolSize=10 2024-11-25T02:54:47,072 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,073 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2c5ea3e41165:0, corePoolSize=2, maxPoolSize=2 2024-11-25T02:54:47,073 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,077 DEBUG [RS:1;2c5ea3e41165:35111 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T02:54:47,077 DEBUG [RS:0;2c5ea3e41165:34269 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T02:54:47,079 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T02:54:47,079 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732503317079 2024-11-25T02:54:47,079 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T02:54:47,081 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T02:54:47,082 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T02:54:47,085 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:47,085 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T02:54:47,086 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T02:54:47,087 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T02:54:47,088 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T02:54:47,088 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T02:54:47,090 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,099 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T02:54:47,100 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T02:54:47,101 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T02:54:47,107 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T02:54:47,107 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T02:54:47,109 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2c5ea3e41165:0:becomeActiveMaster-HFileCleaner.large.0-1732503287109,5,FailOnTimeoutGroup] 2024-11-25T02:54:47,110 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2c5ea3e41165:0:becomeActiveMaster-HFileCleaner.small.0-1732503287110,5,FailOnTimeoutGroup] 2024-11-25T02:54:47,110 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,111 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T02:54:47,111 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,112 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,114 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T02:54:47,114 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a 2024-11-25T02:54:47,136 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T02:54:47,137 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T02:54:47,141 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T02:54:47,141 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:47,142 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:47,142 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T02:54:47,147 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T02:54:47,148 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:47,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:47,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T02:54:47,151 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58765, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T02:54:47,151 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54689, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T02:54:47,154 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T02:54:47,154 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:47,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:47,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T02:54:47,158 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37837 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2c5ea3e41165,34269,1732503285784 2024-11-25T02:54:47,159 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T02:54:47,159 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:47,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:47,159 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T02:54:47,160 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37837 {}] master.ServerManager(517): Registering regionserver=2c5ea3e41165,34269,1732503285784 2024-11-25T02:54:47,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740 2024-11-25T02:54:47,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740 2024-11-25T02:54:47,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T02:54:47,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T02:54:47,162 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T02:54:47,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T02:54:47,170 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T02:54:47,171 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64522774, jitterRate=-0.038535743951797485}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T02:54:47,172 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37837 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2c5ea3e41165,35111,1732503285943 2024-11-25T02:54:47,172 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37837 {}] master.ServerManager(517): Registering regionserver=2c5ea3e41165,35111,1732503285943 2024-11-25T02:54:47,176 DEBUG [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a 2024-11-25T02:54:47,176 DEBUG [RS:1;2c5ea3e41165:35111 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a 2024-11-25T02:54:47,176 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732503287136Initializing all the Stores at 1732503287137 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732503287137Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732503287137Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732503287137Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732503287137Cleaning up temporary data from old regions at 1732503287162 (+25 ms)Region opened successfully at 1732503287176 (+14 ms) 2024-11-25T02:54:47,177 DEBUG [RS:1;2c5ea3e41165:35111 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T02:54:47,177 DEBUG [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T02:54:47,177 DEBUG [RS:1;2c5ea3e41165:35111 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T02:54:47,177 DEBUG [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T02:54:47,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T02:54:47,177 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T02:54:47,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T02:54:47,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T02:54:47,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T02:54:47,178 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T02:54:47,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732503287177Disabling compacts and flushes for region at 1732503287177Disabling writes for close at 1732503287177Writing region close event to WAL at 1732503287178 (+1 ms)Closed at 1732503287178 2024-11-25T02:54:47,180 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T02:54:47,180 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T02:54:47,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T02:54:47,193 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T02:54:47,196 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T02:54:47,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T02:54:47,238 DEBUG [RS:0;2c5ea3e41165:34269 {}] zookeeper.ZKUtil(111): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2c5ea3e41165,34269,1732503285784 2024-11-25T02:54:47,239 WARN [RS:0;2c5ea3e41165:34269 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T02:54:47,239 DEBUG [RS:1;2c5ea3e41165:35111 {}] zookeeper.ZKUtil(111): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2c5ea3e41165,35111,1732503285943 2024-11-25T02:54:47,239 INFO [RS:0;2c5ea3e41165:34269 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T02:54:47,239 WARN [RS:1;2c5ea3e41165:35111 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T02:54:47,239 INFO [RS:1;2c5ea3e41165:35111 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T02:54:47,239 DEBUG [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,34269,1732503285784 2024-11-25T02:54:47,239 DEBUG [RS:1;2c5ea3e41165:35111 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,35111,1732503285943 2024-11-25T02:54:47,240 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2c5ea3e41165,35111,1732503285943] 2024-11-25T02:54:47,240 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2c5ea3e41165,34269,1732503285784] 2024-11-25T02:54:47,264 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T02:54:47,264 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T02:54:47,285 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T02:54:47,285 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T02:54:47,290 INFO [RS:1;2c5ea3e41165:35111 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T02:54:47,290 INFO [RS:0;2c5ea3e41165:34269 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T02:54:47,290 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,290 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,291 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T02:54:47,291 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T02:54:47,296 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T02:54:47,296 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T02:54:47,297 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,297 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,297 DEBUG [RS:0;2c5ea3e41165:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,298 DEBUG [RS:1;2c5ea3e41165:35111 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,298 DEBUG [RS:0;2c5ea3e41165:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,298 DEBUG [RS:1;2c5ea3e41165:35111 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,298 DEBUG [RS:0;2c5ea3e41165:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,298 DEBUG [RS:0;2c5ea3e41165:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,298 DEBUG [RS:1;2c5ea3e41165:35111 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,298 DEBUG [RS:0;2c5ea3e41165:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,298 DEBUG [RS:1;2c5ea3e41165:35111 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,298 DEBUG [RS:0;2c5ea3e41165:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2c5ea3e41165:0, corePoolSize=2, maxPoolSize=2 2024-11-25T02:54:47,298 DEBUG [RS:1;2c5ea3e41165:35111 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,298 DEBUG [RS:0;2c5ea3e41165:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,298 DEBUG [RS:0;2c5ea3e41165:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,299 DEBUG [RS:1;2c5ea3e41165:35111 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2c5ea3e41165:0, corePoolSize=2, maxPoolSize=2 2024-11-25T02:54:47,299 DEBUG [RS:0;2c5ea3e41165:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,299 DEBUG [RS:1;2c5ea3e41165:35111 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,299 DEBUG [RS:0;2c5ea3e41165:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,299 DEBUG [RS:0;2c5ea3e41165:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,299 DEBUG [RS:1;2c5ea3e41165:35111 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,299 DEBUG [RS:0;2c5ea3e41165:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,299 DEBUG [RS:1;2c5ea3e41165:35111 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,299 DEBUG [RS:0;2c5ea3e41165:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2c5ea3e41165:0, corePoolSize=3, maxPoolSize=3 2024-11-25T02:54:47,299 DEBUG [RS:0;2c5ea3e41165:34269 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2c5ea3e41165:0, corePoolSize=3, maxPoolSize=3 2024-11-25T02:54:47,299 DEBUG [RS:1;2c5ea3e41165:35111 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,299 DEBUG [RS:1;2c5ea3e41165:35111 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,300 DEBUG [RS:1;2c5ea3e41165:35111 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:47,300 DEBUG [RS:1;2c5ea3e41165:35111 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2c5ea3e41165:0, corePoolSize=3, maxPoolSize=3 2024-11-25T02:54:47,300 DEBUG [RS:1;2c5ea3e41165:35111 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2c5ea3e41165:0, corePoolSize=3, maxPoolSize=3 2024-11-25T02:54:47,303 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,303 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,303 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,303 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,304 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,304 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,304 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,304 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,304 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,304 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,34269,1732503285784-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T02:54:47,304 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,304 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,35111,1732503285943-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T02:54:47,330 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T02:54:47,330 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T02:54:47,332 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,35111,1732503285943-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,332 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,34269,1732503285784-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,333 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,333 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,333 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.Replication(171): 2c5ea3e41165,35111,1732503285943 started 2024-11-25T02:54:47,333 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.Replication(171): 2c5ea3e41165,34269,1732503285784 started 2024-11-25T02:54:47,347 WARN [2c5ea3e41165:37837 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T02:54:47,357 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,357 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:47,358 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer(1482): Serving as 2c5ea3e41165,34269,1732503285784, RpcServer on 2c5ea3e41165/172.17.0.2:34269, sessionid=0x1017133be700001 2024-11-25T02:54:47,358 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.HRegionServer(1482): Serving as 2c5ea3e41165,35111,1732503285943, RpcServer on 2c5ea3e41165/172.17.0.2:35111, sessionid=0x1017133be700002 2024-11-25T02:54:47,359 DEBUG [RS:1;2c5ea3e41165:35111 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T02:54:47,359 DEBUG [RS:0;2c5ea3e41165:34269 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T02:54:47,359 DEBUG [RS:0;2c5ea3e41165:34269 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2c5ea3e41165,34269,1732503285784 2024-11-25T02:54:47,359 DEBUG [RS:1;2c5ea3e41165:35111 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2c5ea3e41165,35111,1732503285943 2024-11-25T02:54:47,360 DEBUG [RS:0;2c5ea3e41165:34269 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2c5ea3e41165,34269,1732503285784' 2024-11-25T02:54:47,360 DEBUG [RS:1;2c5ea3e41165:35111 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2c5ea3e41165,35111,1732503285943' 2024-11-25T02:54:47,360 DEBUG [RS:1;2c5ea3e41165:35111 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T02:54:47,360 DEBUG [RS:0;2c5ea3e41165:34269 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T02:54:47,361 DEBUG [RS:0;2c5ea3e41165:34269 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T02:54:47,361 DEBUG [RS:1;2c5ea3e41165:35111 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T02:54:47,362 DEBUG [RS:1;2c5ea3e41165:35111 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T02:54:47,362 DEBUG [RS:0;2c5ea3e41165:34269 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T02:54:47,362 DEBUG [RS:1;2c5ea3e41165:35111 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T02:54:47,362 DEBUG [RS:0;2c5ea3e41165:34269 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T02:54:47,362 DEBUG [RS:1;2c5ea3e41165:35111 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2c5ea3e41165,35111,1732503285943 2024-11-25T02:54:47,362 DEBUG [RS:0;2c5ea3e41165:34269 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2c5ea3e41165,34269,1732503285784 2024-11-25T02:54:47,362 DEBUG [RS:1;2c5ea3e41165:35111 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2c5ea3e41165,35111,1732503285943' 2024-11-25T02:54:47,362 DEBUG [RS:0;2c5ea3e41165:34269 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2c5ea3e41165,34269,1732503285784' 2024-11-25T02:54:47,362 DEBUG [RS:1;2c5ea3e41165:35111 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T02:54:47,362 DEBUG [RS:0;2c5ea3e41165:34269 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T02:54:47,363 DEBUG [RS:1;2c5ea3e41165:35111 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T02:54:47,363 DEBUG [RS:0;2c5ea3e41165:34269 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T02:54:47,363 DEBUG [RS:1;2c5ea3e41165:35111 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T02:54:47,363 INFO [RS:1;2c5ea3e41165:35111 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T02:54:47,363 DEBUG [RS:0;2c5ea3e41165:34269 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T02:54:47,364 INFO [RS:1;2c5ea3e41165:35111 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T02:54:47,364 INFO [RS:0;2c5ea3e41165:34269 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T02:54:47,364 INFO [RS:0;2c5ea3e41165:34269 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T02:54:47,475 INFO [RS:0;2c5ea3e41165:34269 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T02:54:47,475 INFO [RS:1;2c5ea3e41165:35111 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T02:54:47,477 INFO [RS:1;2c5ea3e41165:35111 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2c5ea3e41165%2C35111%2C1732503285943, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,35111,1732503285943, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/oldWALs, maxLogs=55 2024-11-25T02:54:47,478 INFO [RS:0;2c5ea3e41165:34269 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2c5ea3e41165%2C34269%2C1732503285784, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,34269,1732503285784, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/oldWALs, maxLogs=55 2024-11-25T02:54:47,500 INFO [RS:1;2c5ea3e41165:35111 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,35111,1732503285943/2c5ea3e41165%2C35111%2C1732503285943.1732503287481 2024-11-25T02:54:47,500 INFO [RS:0;2c5ea3e41165:34269 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,34269,1732503285784/2c5ea3e41165%2C34269%2C1732503285784.1732503287481 2024-11-25T02:54:47,500 DEBUG [RS:1;2c5ea3e41165:35111 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T02:54:47,503 DEBUG [RS:0;2c5ea3e41165:34269 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T02:54:47,601 DEBUG [2c5ea3e41165:37837 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-25T02:54:47,614 DEBUG [2c5ea3e41165:37837 {}] balancer.BalancerClusterState(204): Hosts are {2c5ea3e41165=0} racks are {/default-rack=0} 2024-11-25T02:54:47,617 DEBUG [2c5ea3e41165:37837 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T02:54:47,618 DEBUG [2c5ea3e41165:37837 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T02:54:47,618 DEBUG [2c5ea3e41165:37837 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T02:54:47,618 DEBUG [2c5ea3e41165:37837 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T02:54:47,618 INFO [2c5ea3e41165:37837 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T02:54:47,618 INFO [2c5ea3e41165:37837 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T02:54:47,618 DEBUG [2c5ea3e41165:37837 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T02:54:47,625 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2c5ea3e41165,34269,1732503285784 2024-11-25T02:54:47,630 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2c5ea3e41165,34269,1732503285784, state=OPENING 2024-11-25T02:54:47,664 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T02:54:47,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:47,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:47,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:47,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T02:54:47,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T02:54:47,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T02:54:47,727 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T02:54:47,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2c5ea3e41165,34269,1732503285784}] 2024-11-25T02:54:47,904 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T02:54:47,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34865, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T02:54:47,915 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T02:54:47,915 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T02:54:47,916 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T02:54:47,918 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2c5ea3e41165%2C34269%2C1732503285784.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,34269,1732503285784, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/oldWALs, maxLogs=55 2024-11-25T02:54:47,940 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,34269,1732503285784/2c5ea3e41165%2C34269%2C1732503285784.meta.1732503287921.meta 2024-11-25T02:54:47,940 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T02:54:47,940 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T02:54:47,942 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T02:54:47,944 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T02:54:47,949 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T02:54:47,952 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T02:54:47,953 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T02:54:47,953 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T02:54:47,953 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T02:54:47,955 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T02:54:47,956 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T02:54:47,956 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:47,956 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:47,956 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T02:54:47,957 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T02:54:47,957 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:47,957 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:47,958 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T02:54:47,958 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T02:54:47,958 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:47,959 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:47,959 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T02:54:47,959 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T02:54:47,960 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:47,960 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:47,960 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T02:54:47,961 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740 2024-11-25T02:54:47,962 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740 2024-11-25T02:54:47,962 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T02:54:47,962 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T02:54:47,962 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T02:54:47,963 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T02:54:47,964 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72964485, jitterRate=0.0872555524110794}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T02:54:47,964 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T02:54:47,965 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732503287954Writing region info on filesystem at 1732503287954Initializing all the Stores at 1732503287954Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732503287954Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732503287955 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732503287955Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732503287955Cleaning up temporary data from old regions at 1732503287962 (+7 ms)Running coprocessor post-open hooks at 1732503287964 (+2 ms)Region opened successfully at 1732503287965 (+1 ms) 2024-11-25T02:54:47,971 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732503287897 2024-11-25T02:54:47,980 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T02:54:47,981 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T02:54:47,982 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2c5ea3e41165,34269,1732503285784 2024-11-25T02:54:47,983 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2c5ea3e41165,34269,1732503285784, state=OPEN 2024-11-25T02:54:48,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T02:54:48,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T02:54:48,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T02:54:48,008 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T02:54:48,008 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T02:54:48,008 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T02:54:48,009 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2c5ea3e41165,34269,1732503285784 2024-11-25T02:54:48,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T02:54:48,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2c5ea3e41165,34269,1732503285784 in 280 msec 2024-11-25T02:54:48,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T02:54:48,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 831 msec 2024-11-25T02:54:48,021 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T02:54:48,021 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T02:54:48,044 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T02:54:48,045 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2c5ea3e41165,34269,1732503285784, seqNum=-1] 2024-11-25T02:54:48,062 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T02:54:48,065 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52845, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T02:54:48,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0960 sec 2024-11-25T02:54:48,082 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732503288082, completionTime=-1 2024-11-25T02:54:48,085 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-25T02:54:48,085 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T02:54:48,109 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T02:54:48,109 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732503348109 2024-11-25T02:54:48,109 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732503408109 2024-11-25T02:54:48,110 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-25T02:54:48,112 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-25T02:54:48,118 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,37837,1732503284943-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:48,118 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,37837,1732503284943-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:48,118 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,37837,1732503284943-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:48,120 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2c5ea3e41165:37837, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:48,120 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:48,121 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:48,126 DEBUG [master/2c5ea3e41165:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T02:54:48,151 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.068sec 2024-11-25T02:54:48,153 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T02:54:48,154 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T02:54:48,155 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T02:54:48,167 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T02:54:48,168 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T02:54:48,169 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,37837,1732503284943-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T02:54:48,170 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,37837,1732503284943-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T02:54:48,173 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T02:54:48,174 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T02:54:48,175 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,37837,1732503284943-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:48,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109a9e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T02:54:48,208 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T02:54:48,208 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T02:54:48,212 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2c5ea3e41165,37837,-1 for getting cluster id 2024-11-25T02:54:48,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T02:54:48,221 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '41bb3781-782d-416c-a0f1-7020b486cc6d' 2024-11-25T02:54:48,223 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T02:54:48,224 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "41bb3781-782d-416c-a0f1-7020b486cc6d" 2024-11-25T02:54:48,225 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a78b7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T02:54:48,225 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2c5ea3e41165,37837,-1] 2024-11-25T02:54:48,228 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T02:54:48,229 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:48,231 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T02:54:48,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55969d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T02:54:48,234 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T02:54:48,240 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2c5ea3e41165,34269,1732503285784, seqNum=-1] 2024-11-25T02:54:48,240 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T02:54:48,243 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T02:54:48,262 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2c5ea3e41165,37837,1732503284943 2024-11-25T02:54:48,265 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T02:54:48,269 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2c5ea3e41165,37837,1732503284943 2024-11-25T02:54:48,271 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2d5a7858 2024-11-25T02:54:48,272 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T02:54:48,275 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T02:54:48,281 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37837 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732503288262', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T02:54:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37837 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732503288262 2024-11-25T02:54:48,291 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732503288262 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T02:54:48,294 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37837 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732503288262" procId is: 4 2024-11-25T02:54:48,294 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:48,296 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732503288262 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T02:54:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T02:54:48,308 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732503288262-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1494e4dd7b9d8f9b2498fbf7fc3f6750, NAME => 'testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732503288262', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a 2024-11-25T02:54:48,315 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732503288262-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T02:54:48,315 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732503288262-pool-0 {}] regionserver.HRegion(1722): Closing 1494e4dd7b9d8f9b2498fbf7fc3f6750, disabling compactions & flushes 2024-11-25T02:54:48,315 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732503288262-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:48,315 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732503288262-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:48,315 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732503288262-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. after waiting 0 ms 2024-11-25T02:54:48,315 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732503288262-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:48,315 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732503288262-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:48,315 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732503288262-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1494e4dd7b9d8f9b2498fbf7fc3f6750: Waiting for close lock at 1732503288315Disabling compacts and flushes for region at 1732503288315Disabling writes for close at 1732503288315Writing region close event to WAL at 1732503288315Closed at 1732503288315 2024-11-25T02:54:48,317 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732503288262 execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T02:54:48,322 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732503288317"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732503288317"}]},"ts":"1732503288317"} 2024-11-25T02:54:48,326 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T02:54:48,327 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732503288262 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T02:54:48,330 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732503288262","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732503288327"}]},"ts":"1732503288327"} 2024-11-25T02:54:48,334 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732503288262, state=ENABLING in hbase:meta 2024-11-25T02:54:48,334 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {2c5ea3e41165=0} racks are {/default-rack=0} 2024-11-25T02:54:48,335 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T02:54:48,335 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T02:54:48,335 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T02:54:48,335 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T02:54:48,335 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T02:54:48,335 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T02:54:48,335 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T02:54:48,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732503288262, region=1494e4dd7b9d8f9b2498fbf7fc3f6750, ASSIGN}] 2024-11-25T02:54:48,338 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732503288262, region=1494e4dd7b9d8f9b2498fbf7fc3f6750, ASSIGN 2024-11-25T02:54:48,339 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732503288262, region=1494e4dd7b9d8f9b2498fbf7fc3f6750, ASSIGN; state=OFFLINE, location=2c5ea3e41165,35111,1732503285943; forceNewPlan=false, retain=false 2024-11-25T02:54:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T02:54:48,494 INFO [2c5ea3e41165:37837 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T02:54:48,495 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1494e4dd7b9d8f9b2498fbf7fc3f6750, regionState=OPENING, regionLocation=2c5ea3e41165,35111,1732503285943 2024-11-25T02:54:48,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732503288262, region=1494e4dd7b9d8f9b2498fbf7fc3f6750, ASSIGN because future has completed 2024-11-25T02:54:48,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1494e4dd7b9d8f9b2498fbf7fc3f6750, server=2c5ea3e41165,35111,1732503285943}] 2024-11-25T02:54:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T02:54:48,654 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T02:54:48,656 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44299, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T02:54:48,659 INFO [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:48,659 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1494e4dd7b9d8f9b2498fbf7fc3f6750, NAME => 'testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750.', STARTKEY => '', ENDKEY => ''} 2024-11-25T02:54:48,660 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732503288262 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:48,660 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T02:54:48,660 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:48,660 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:48,661 INFO [StoreOpener-1494e4dd7b9d8f9b2498fbf7fc3f6750-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:48,666 INFO [StoreOpener-1494e4dd7b9d8f9b2498fbf7fc3f6750-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1494e4dd7b9d8f9b2498fbf7fc3f6750 columnFamilyName family 2024-11-25T02:54:48,666 DEBUG [StoreOpener-1494e4dd7b9d8f9b2498fbf7fc3f6750-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:48,666 INFO [StoreOpener-1494e4dd7b9d8f9b2498fbf7fc3f6750-1 {}] regionserver.HStore(327): Store=1494e4dd7b9d8f9b2498fbf7fc3f6750/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T02:54:48,667 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:48,668 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:48,668 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:48,668 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:48,668 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:48,670 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:48,675 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T02:54:48,675 INFO [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1494e4dd7b9d8f9b2498fbf7fc3f6750; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66483334, jitterRate=-0.009321123361587524}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T02:54:48,675 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:48,676 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1494e4dd7b9d8f9b2498fbf7fc3f6750: Running coprocessor pre-open hook at 1732503288660Writing region info on filesystem at 1732503288660Initializing all the Stores at 1732503288661 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732503288661Cleaning up temporary data from old regions at 1732503288668 (+7 ms)Running coprocessor post-open hooks at 1732503288675 (+7 ms)Region opened successfully at 1732503288676 (+1 ms) 2024-11-25T02:54:48,677 INFO [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750., pid=6, masterSystemTime=1732503288653 2024-11-25T02:54:48,680 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:48,681 INFO [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:48,681 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1494e4dd7b9d8f9b2498fbf7fc3f6750, regionState=OPEN, openSeqNum=2, regionLocation=2c5ea3e41165,35111,1732503285943 2024-11-25T02:54:48,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1494e4dd7b9d8f9b2498fbf7fc3f6750, server=2c5ea3e41165,35111,1732503285943 because future has completed 2024-11-25T02:54:48,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T02:54:48,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1494e4dd7b9d8f9b2498fbf7fc3f6750, server=2c5ea3e41165,35111,1732503285943 in 185 msec 2024-11-25T02:54:48,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T02:54:48,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732503288262, region=1494e4dd7b9d8f9b2498fbf7fc3f6750, ASSIGN in 352 msec 2024-11-25T02:54:48,691 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732503288262 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T02:54:48,691 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732503288262","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732503288691"}]},"ts":"1732503288691"} 2024-11-25T02:54:48,694 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732503288262, state=ENABLED in hbase:meta 2024-11-25T02:54:48,695 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732503288262 execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T02:54:48,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732503288262 in 410 msec 2024-11-25T02:54:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37837 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T02:54:48,929 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732503288262 get assigned. Timeout = 60000ms 2024-11-25T02:54:48,929 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732503288262 completed 2024-11-25T02:54:48,932 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T02:54:48,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732503288262 assigned to meta. Checking AM states. 2024-11-25T02:54:48,942 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T02:54:48,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732503288262 assigned. 2024-11-25T02:54:48,953 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732503288262', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750., hostname=2c5ea3e41165,35111,1732503285943, seqNum=2] 2024-11-25T02:54:48,954 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T02:54:48,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T02:54:48,967 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1494e4dd7b9d8f9b2498fbf7fc3f6750 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-25T02:54:48,991 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4df0a6573b41405ba6a17f0a27a49fc6 2024-11-25T02:54:49,030 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/.tmp/family/4df0a6573b41405ba6a17f0a27a49fc6 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-25T02:54:49,035 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/.tmp/family/4df0a6573b41405ba6a17f0a27a49fc6 2024-11-25T02:54:49,065 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/.tmp/family/4df0a6573b41405ba6a17f0a27a49fc6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/family/4df0a6573b41405ba6a17f0a27a49fc6 2024-11-25T02:54:49,066 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/family/4df0a6573b41405ba6a17f0a27a49fc6, delay=905 ms 2024-11-25T02:54:49,068 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/family/4df0a6573b41405ba6a17f0a27a49fc6, entries=2, sequenceid=6, filesize=4.9 K 2024-11-25T02:54:49,073 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 1494e4dd7b9d8f9b2498fbf7fc3f6750 in 106ms, sequenceid=6, compaction requested=false 2024-11-25T02:54:49,074 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732503288262' 2024-11-25T02:54:49,076 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1494e4dd7b9d8f9b2498fbf7fc3f6750: 2024-11-25T02:54:49,976 DEBUG [hfile-prefetch-1732503289066 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4df0a6573b41405ba6a17f0a27a49fc6_0. 2024-11-25T02:54:49,983 DEBUG [hfile-prefetch-1732503289066 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4df0a6573b41405ba6a17f0a27a49fc6_117. 2024-11-25T02:54:49,984 DEBUG [hfile-prefetch-1732503289066 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/family/4df0a6573b41405ba6a17f0a27a49fc6, with total blocks 2, and data blocks 1 2024-11-25T02:54:49,984 DEBUG [hfile-prefetch-1732503289066 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-25T02:54:49,985 DEBUG [hfile-prefetch-1732503289066 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/family/4df0a6573b41405ba6a17f0a27a49fc6, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-25T02:54:49,985 DEBUG [hfile-prefetch-1732503289066 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/family/4df0a6573b41405ba6a17f0a27a49fc6. Will sleep 100ms and try the verification again. 2024-11-25T02:54:50,082 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T02:54:50,083 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T02:54:50,084 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T02:54:50,084 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T02:54:50,085 DEBUG [hfile-prefetch-1732503289066 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/family/4df0a6573b41405ba6a17f0a27a49fc6, with total blocks 2, and data blocks 1 2024-11-25T02:54:50,085 DEBUG [hfile-prefetch-1732503289066 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-25T02:54:50,086 DEBUG [hfile-prefetch-1732503289066 {}] bucket.BucketCache(2293): found block for file 4df0a6573b41405ba6a17f0a27a49fc6 in the backing map. Acquiring read lock for offset 0 2024-11-25T02:54:50,086 DEBUG [hfile-prefetch-1732503289066 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/family/4df0a6573b41405ba6a17f0a27a49fc6 has now been fully cached. 2024-11-25T02:54:50,087 DEBUG [hfile-prefetch-1732503289066 {}] hfile.PrefetchExecutor(121): Prefetch completed for 4df0a6573b41405ba6a17f0a27a49fc6 2024-11-25T02:54:50,091 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T02:54:50,091 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T02:54:50,091 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T02:54:50,091 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T02:54:50,094 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37837 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=1494e4dd7b9d8f9b2498fbf7fc3f6750, source=2c5ea3e41165,35111,1732503285943, destination=2c5ea3e41165,34269,1732503285784, warming up region on 2c5ea3e41165,34269,1732503285784 2024-11-25T02:54:50,096 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37837 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=1494e4dd7b9d8f9b2498fbf7fc3f6750, source=2c5ea3e41165,35111,1732503285943, destination=2c5ea3e41165,34269,1732503285784, running balancer 2024-11-25T02:54:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37837 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732503288262, region=1494e4dd7b9d8f9b2498fbf7fc3f6750, REOPEN/MOVE 2024-11-25T02:54:50,098 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732503288262, region=1494e4dd7b9d8f9b2498fbf7fc3f6750, REOPEN/MOVE 2024-11-25T02:54:50,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(7855): Warmup {ENCODED => 1494e4dd7b9d8f9b2498fbf7fc3f6750, NAME => 'testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750.', STARTKEY => '', ENDKEY => ''} 2024-11-25T02:54:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T02:54:50,100 INFO [StoreOpener-1494e4dd7b9d8f9b2498fbf7fc3f6750-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:50,101 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1494e4dd7b9d8f9b2498fbf7fc3f6750, regionState=CLOSING, regionLocation=2c5ea3e41165,35111,1732503285943 2024-11-25T02:54:50,101 INFO [StoreOpener-1494e4dd7b9d8f9b2498fbf7fc3f6750-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1494e4dd7b9d8f9b2498fbf7fc3f6750 columnFamilyName family 2024-11-25T02:54:50,101 DEBUG [StoreOpener-1494e4dd7b9d8f9b2498fbf7fc3f6750-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:50,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732503288262, region=1494e4dd7b9d8f9b2498fbf7fc3f6750, REOPEN/MOVE because future has completed 2024-11-25T02:54:50,106 DEBUG [StoreOpener-1494e4dd7b9d8f9b2498fbf7fc3f6750-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/family/4df0a6573b41405ba6a17f0a27a49fc6 2024-11-25T02:54:50,106 INFO [StoreOpener-1494e4dd7b9d8f9b2498fbf7fc3f6750-1 {}] regionserver.HStore(327): Store=1494e4dd7b9d8f9b2498fbf7fc3f6750/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T02:54:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(1722): Closing 1494e4dd7b9d8f9b2498fbf7fc3f6750, disabling compactions & flushes 2024-11-25T02:54:50,107 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. after waiting 0 ms 2024-11-25T02:54:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,108 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-25T02:54:50,108 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1494e4dd7b9d8f9b2498fbf7fc3f6750, server=2c5ea3e41165,35111,1732503285943}] 2024-11-25T02:54:50,108 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T02:54:50,108 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732503290091 2024-11-25T02:54:50,108 WARN [Thread-89 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-25T02:54:50,108 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T02:54:50,108 ERROR [bucket-cache-persister {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2024-11-25T02:54:50,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34269 {}] regionserver.HRegion(1676): Region close journal for 1494e4dd7b9d8f9b2498fbf7fc3f6750: Waiting for close lock at 1732503290106Disabling compacts and flushes for region at 1732503290106Disabling writes for close at 1732503290107 (+1 ms)Writing region close event to WAL at 1732503290110 (+3 ms)Closed at 1732503290110 2024-11-25T02:54:50,112 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-25T02:54:50,113 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732503290083 2024-11-25T02:54:50,113 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T02:54:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37837 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-25T02:54:50,267 INFO [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:50,268 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-25T02:54:50,268 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 1494e4dd7b9d8f9b2498fbf7fc3f6750, disabling compactions & flushes 2024-11-25T02:54:50,268 INFO [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,268 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,268 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. after waiting 0 ms 2024-11-25T02:54:50,268 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,270 DEBUG [StoreFileCloser-1494e4dd7b9d8f9b2498fbf7fc3f6750-family-1 {}] bucket.BucketCache(768): Removing file 4df0a6573b41405ba6a17f0a27a49fc6 for region 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:50,270 DEBUG [StoreFileCloser-1494e4dd7b9d8f9b2498fbf7fc3f6750-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 4df0a6573b41405ba6a17f0a27a49fc6, starting offset: 0, end offset: 9223372036854775807 2024-11-25T02:54:50,271 DEBUG [StoreFileCloser-1494e4dd7b9d8f9b2498fbf7fc3f6750-family-1 {}] bucket.BucketCache(856): removed key 4df0a6573b41405ba6a17f0a27a49fc6_0 from back map with offset lock 2094080 in the evict process 2024-11-25T02:54:50,276 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-25T02:54:50,277 INFO [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,277 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 1494e4dd7b9d8f9b2498fbf7fc3f6750: Waiting for close lock at 1732503290268Running coprocessor pre-close hooks at 1732503290268Disabling compacts and flushes for region at 1732503290268Disabling writes for close at 1732503290268Writing region close event to WAL at 1732503290272 (+4 ms)Running coprocessor post-close hooks at 1732503290276 (+4 ms)Closed at 1732503290277 (+1 ms) 2024-11-25T02:54:50,277 INFO [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 1494e4dd7b9d8f9b2498fbf7fc3f6750 move to 2c5ea3e41165,34269,1732503285784 record at close sequenceid=6 2024-11-25T02:54:50,281 INFO [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:50,281 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1494e4dd7b9d8f9b2498fbf7fc3f6750, regionState=CLOSED 2024-11-25T02:54:50,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1494e4dd7b9d8f9b2498fbf7fc3f6750, server=2c5ea3e41165,35111,1732503285943 because future has completed 2024-11-25T02:54:50,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T02:54:50,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 1494e4dd7b9d8f9b2498fbf7fc3f6750, server=2c5ea3e41165,35111,1732503285943 in 178 msec 2024-11-25T02:54:50,289 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732503288262, region=1494e4dd7b9d8f9b2498fbf7fc3f6750, REOPEN/MOVE; state=CLOSED, location=2c5ea3e41165,34269,1732503285784; forceNewPlan=false, retain=false 2024-11-25T02:54:50,309 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T02:54:50,309 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T02:54:50,309 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T02:54:50,309 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T02:54:50,312 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T02:54:50,312 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732503290309 2024-11-25T02:54:50,312 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T02:54:50,440 INFO [2c5ea3e41165:37837 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T02:54:50,441 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1494e4dd7b9d8f9b2498fbf7fc3f6750, regionState=OPENING, regionLocation=2c5ea3e41165,34269,1732503285784 2024-11-25T02:54:50,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732503288262, region=1494e4dd7b9d8f9b2498fbf7fc3f6750, REOPEN/MOVE because future has completed 2024-11-25T02:54:50,448 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1494e4dd7b9d8f9b2498fbf7fc3f6750, server=2c5ea3e41165,34269,1732503285784}] 2024-11-25T02:54:50,605 INFO [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,605 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1494e4dd7b9d8f9b2498fbf7fc3f6750, NAME => 'testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750.', STARTKEY => '', ENDKEY => ''} 2024-11-25T02:54:50,607 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732503288262 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:50,607 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T02:54:50,607 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:50,607 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:50,609 INFO [StoreOpener-1494e4dd7b9d8f9b2498fbf7fc3f6750-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:50,611 INFO [StoreOpener-1494e4dd7b9d8f9b2498fbf7fc3f6750-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1494e4dd7b9d8f9b2498fbf7fc3f6750 columnFamilyName family 2024-11-25T02:54:50,611 DEBUG [StoreOpener-1494e4dd7b9d8f9b2498fbf7fc3f6750-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:50,615 DEBUG [StoreFileOpener-1494e4dd7b9d8f9b2498fbf7fc3f6750-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/family/4df0a6573b41405ba6a17f0a27a49fc6, delay=916 ms 2024-11-25T02:54:50,616 DEBUG [StoreOpener-1494e4dd7b9d8f9b2498fbf7fc3f6750-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/family/4df0a6573b41405ba6a17f0a27a49fc6 2024-11-25T02:54:50,617 INFO [StoreOpener-1494e4dd7b9d8f9b2498fbf7fc3f6750-1 {}] regionserver.HStore(327): Store=1494e4dd7b9d8f9b2498fbf7fc3f6750/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T02:54:50,617 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:50,619 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:50,619 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:50,620 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:50,620 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:50,626 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:50,627 INFO [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 1494e4dd7b9d8f9b2498fbf7fc3f6750; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60155879, jitterRate=-0.10360755026340485}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T02:54:50,627 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:50,628 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 1494e4dd7b9d8f9b2498fbf7fc3f6750: Running coprocessor pre-open hook at 1732503290608Writing region info on filesystem at 1732503290608Initializing all the Stores at 1732503290608Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732503290608Cleaning up temporary data from old regions at 1732503290620 (+12 ms)Running coprocessor post-open hooks at 1732503290627 (+7 ms)Region opened successfully at 1732503290628 (+1 ms) 2024-11-25T02:54:50,629 INFO [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750., pid=9, masterSystemTime=1732503290601 2024-11-25T02:54:50,632 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,632 INFO [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,632 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1494e4dd7b9d8f9b2498fbf7fc3f6750, regionState=OPEN, openSeqNum=10, regionLocation=2c5ea3e41165,34269,1732503285784 2024-11-25T02:54:50,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1494e4dd7b9d8f9b2498fbf7fc3f6750, server=2c5ea3e41165,34269,1732503285784 because future has completed 2024-11-25T02:54:50,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-25T02:54:50,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 1494e4dd7b9d8f9b2498fbf7fc3f6750, server=2c5ea3e41165,34269,1732503285784 in 188 msec 2024-11-25T02:54:50,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732503288262, region=1494e4dd7b9d8f9b2498fbf7fc3f6750, REOPEN/MOVE in 542 msec 2024-11-25T02:54:50,652 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T02:54:50,652 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T02:54:50,653 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T02:54:50,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:50,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:50,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T02:54:50,657 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T02:54:50,657 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1692727640, stopped=false 2024-11-25T02:54:50,658 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2c5ea3e41165,37837,1732503284943 2024-11-25T02:54:50,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T02:54:50,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T02:54:50,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T02:54:50,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:50,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:50,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:50,682 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T02:54:50,683 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T02:54:50,683 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T02:54:50,684 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T02:54:50,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:50,684 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T02:54:50,684 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T02:54:50,685 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2c5ea3e41165,34269,1732503285784' ***** 2024-11-25T02:54:50,685 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T02:54:50,685 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2c5ea3e41165,35111,1732503285943' ***** 2024-11-25T02:54:50,685 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T02:54:50,686 INFO [RS:0;2c5ea3e41165:34269 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T02:54:50,686 INFO [RS:1;2c5ea3e41165:35111 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T02:54:50,686 INFO [RS:0;2c5ea3e41165:34269 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/bucket.persistence 2024-11-25T02:54:50,686 INFO [RS:1;2c5ea3e41165:35111 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/bucket.persistence 2024-11-25T02:54:50,686 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T02:54:50,686 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T02:54:50,686 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T02:54:50,687 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T02:54:50,687 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T02:54:50,687 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T02:54:50,687 INFO [RS:1;2c5ea3e41165:35111 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T02:54:50,687 INFO [RS:0;2c5ea3e41165:34269 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T02:54:50,687 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T02:54:50,687 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T02:54:50,697 DEBUG [RS:1;2c5ea3e41165:35111 {}] bucket.BucketCache(1487): Thread RS:1;2c5ea3e41165:35111 started persisting bucket cache to file 2024-11-25T02:54:50,697 DEBUG [RS:0;2c5ea3e41165:34269 {}] bucket.BucketCache(1487): Thread RS:0;2c5ea3e41165:34269 started persisting bucket cache to file 2024-11-25T02:54:50,697 DEBUG [RS:1;2c5ea3e41165:35111 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T02:54:50,697 DEBUG [RS:0;2c5ea3e41165:34269 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T02:54:50,697 DEBUG [RS:1;2c5ea3e41165:35111 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T02:54:50,697 DEBUG [RS:0;2c5ea3e41165:34269 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T02:54:50,701 DEBUG [RS:1;2c5ea3e41165:35111 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T02:54:50,701 DEBUG [RS:1;2c5ea3e41165:35111 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732503290697 2024-11-25T02:54:50,701 DEBUG [RS:0;2c5ea3e41165:34269 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T02:54:50,701 DEBUG [RS:0;2c5ea3e41165:34269 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732503290697 2024-11-25T02:54:50,701 DEBUG [RS:0;2c5ea3e41165:34269 {}] bucket.BucketCache(1509): Thread RS:0;2c5ea3e41165:34269 finished persisting bucket cache to file, renaming 2024-11-25T02:54:50,701 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T02:54:50,702 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T02:54:50,702 INFO [RS:0;2c5ea3e41165:34269 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T02:54:50,702 INFO [RS:0;2c5ea3e41165:34269 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T02:54:50,702 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer(3091): Received CLOSE for 1494e4dd7b9d8f9b2498fbf7fc3f6750 2024-11-25T02:54:50,703 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer(959): stopping server 2c5ea3e41165,34269,1732503285784 2024-11-25T02:54:50,703 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T02:54:50,703 INFO [RS:0;2c5ea3e41165:34269 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2c5ea3e41165:34269. 2024-11-25T02:54:50,703 DEBUG [RS:0;2c5ea3e41165:34269 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T02:54:50,703 DEBUG [RS:0;2c5ea3e41165:34269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:50,703 INFO [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T02:54:50,703 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T02:54:50,703 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1494e4dd7b9d8f9b2498fbf7fc3f6750, disabling compactions & flushes 2024-11-25T02:54:50,703 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T02:54:50,703 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T02:54:50,703 INFO [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,703 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T02:54:50,703 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,704 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. after waiting 0 ms 2024-11-25T02:54:50,704 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,704 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-25T02:54:50,704 DEBUG [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 1494e4dd7b9d8f9b2498fbf7fc3f6750=testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750.} 2024-11-25T02:54:50,704 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T02:54:50,704 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T02:54:50,704 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T02:54:50,704 DEBUG [StoreFileCloser-1494e4dd7b9d8f9b2498fbf7fc3f6750-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/family/4df0a6573b41405ba6a17f0a27a49fc6 2024-11-25T02:54:50,704 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T02:54:50,704 DEBUG [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer(1351): Waiting on 1494e4dd7b9d8f9b2498fbf7fc3f6750, 1588230740 2024-11-25T02:54:50,704 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T02:54:50,705 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T02:54:50,705 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-25T02:54:50,707 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T02:54:50,709 INFO [regionserver/2c5ea3e41165:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T02:54:50,709 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnRegionMove1732503288262/1494e4dd7b9d8f9b2498fbf7fc3f6750/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-25T02:54:50,709 INFO [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,710 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1494e4dd7b9d8f9b2498fbf7fc3f6750: Waiting for close lock at 1732503290703Running coprocessor pre-close hooks at 1732503290703Disabling compacts and flushes for region at 1732503290703Disabling writes for close at 1732503290704 (+1 ms)Writing region close event to WAL at 1732503290705 (+1 ms)Running coprocessor post-close hooks at 1732503290709 (+4 ms)Closed at 1732503290709 2024-11-25T02:54:50,710 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750. 2024-11-25T02:54:50,711 INFO [regionserver/2c5ea3e41165:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T02:54:50,713 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d508aeb56d314f0c8fa076ccf2279643 2024-11-25T02:54:50,714 DEBUG [RS:1;2c5ea3e41165:35111 {}] bucket.BucketCache(1509): Thread RS:1;2c5ea3e41165:35111 finished persisting bucket cache to file, renaming 2024-11-25T02:54:50,714 WARN [RS:1;2c5ea3e41165:35111 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-25T02:54:50,714 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T02:54:50,714 INFO [RS:1;2c5ea3e41165:35111 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T02:54:50,714 INFO [RS:1;2c5ea3e41165:35111 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T02:54:50,714 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.HRegionServer(959): stopping server 2c5ea3e41165,35111,1732503285943 2024-11-25T02:54:50,714 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T02:54:50,714 INFO [RS:1;2c5ea3e41165:35111 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2c5ea3e41165:35111. 2024-11-25T02:54:50,714 DEBUG [RS:1;2c5ea3e41165:35111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T02:54:50,714 DEBUG [RS:1;2c5ea3e41165:35111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:50,715 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.HRegionServer(976): stopping server 2c5ea3e41165,35111,1732503285943; all regions closed. 2024-11-25T02:54:50,719 DEBUG [RS:1;2c5ea3e41165:35111 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/oldWALs 2024-11-25T02:54:50,719 INFO [RS:1;2c5ea3e41165:35111 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2c5ea3e41165%2C35111%2C1732503285943:(num 1732503287481) 2024-11-25T02:54:50,719 DEBUG [RS:1;2c5ea3e41165:35111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:50,719 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T02:54:50,719 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T02:54:50,719 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.ChoreService(370): Chore service for: regionserver/2c5ea3e41165:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T02:54:50,720 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T02:54:50,720 INFO [regionserver/2c5ea3e41165:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T02:54:50,720 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T02:54:50,720 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T02:54:50,720 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T02:54:50,720 INFO [RS:1;2c5ea3e41165:35111 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35111 2024-11-25T02:54:50,730 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/info/d508aeb56d314f0c8fa076ccf2279643 is 205, key is testBlockEvictionOnRegionMove1732503288262,,1732503288276.1494e4dd7b9d8f9b2498fbf7fc3f6750./info:regioninfo/1732503290632/Put/seqid=0 2024-11-25T02:54:50,731 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/info/d508aeb56d314f0c8fa076ccf2279643 2024-11-25T02:54:50,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2c5ea3e41165,35111,1732503285943 2024-11-25T02:54:50,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T02:54:50,740 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c9ce3eff2ebe42df8809bec5327b6378 2024-11-25T02:54:50,740 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T02:54:50,749 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2c5ea3e41165,35111,1732503285943] 2024-11-25T02:54:50,753 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/ns/c9ce3eff2ebe42df8809bec5327b6378 is 43, key is default/ns:d/1732503288069/Put/seqid=0 2024-11-25T02:54:50,754 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/ns/c9ce3eff2ebe42df8809bec5327b6378 2024-11-25T02:54:50,757 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2c5ea3e41165,35111,1732503285943 already deleted, retry=false 2024-11-25T02:54:50,757 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2c5ea3e41165,35111,1732503285943 expired; onlineServers=1 2024-11-25T02:54:50,762 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8361d0000b564140a1305dce1a793dde 2024-11-25T02:54:50,774 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/table/8361d0000b564140a1305dce1a793dde is 78, key is testBlockEvictionOnRegionMove1732503288262/table:state/1732503288691/Put/seqid=0 2024-11-25T02:54:50,775 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/table/8361d0000b564140a1305dce1a793dde 2024-11-25T02:54:50,777 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/info/d508aeb56d314f0c8fa076ccf2279643 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/info/d508aeb56d314f0c8fa076ccf2279643 2024-11-25T02:54:50,778 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/info/d508aeb56d314f0c8fa076ccf2279643, delay=968 ms 2024-11-25T02:54:50,780 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/info/d508aeb56d314f0c8fa076ccf2279643, entries=15, sequenceid=15, filesize=7.6 K 2024-11-25T02:54:50,780 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/ns/c9ce3eff2ebe42df8809bec5327b6378 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/ns/c9ce3eff2ebe42df8809bec5327b6378 2024-11-25T02:54:50,781 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/ns/c9ce3eff2ebe42df8809bec5327b6378, delay=969 ms 2024-11-25T02:54:50,782 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/ns/c9ce3eff2ebe42df8809bec5327b6378, entries=2, sequenceid=15, filesize=5.0 K 2024-11-25T02:54:50,783 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/table/8361d0000b564140a1305dce1a793dde as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/table/8361d0000b564140a1305dce1a793dde 2024-11-25T02:54:50,784 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/table/8361d0000b564140a1305dce1a793dde, delay=904 ms 2024-11-25T02:54:50,785 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/table/8361d0000b564140a1305dce1a793dde, entries=2, sequenceid=15, filesize=5.3 K 2024-11-25T02:54:50,786 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=15, compaction requested=false 2024-11-25T02:54:50,786 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T02:54:50,787 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/info/d508aeb56d314f0c8fa076ccf2279643 2024-11-25T02:54:50,788 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/ns/c9ce3eff2ebe42df8809bec5327b6378 2024-11-25T02:54:50,788 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/table/8361d0000b564140a1305dce1a793dde 2024-11-25T02:54:50,792 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-25T02:54:50,792 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T02:54:50,793 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T02:54:50,793 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732503290704Running coprocessor pre-close hooks at 1732503290704Disabling compacts and flushes for region at 1732503290704Disabling writes for close at 1732503290704Obtaining lock to block concurrent updates at 1732503290705 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732503290705Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732503290705Flushing stores of hbase:meta,,1.1588230740 at 1732503290706 (+1 ms)Flushing 1588230740/info: creating writer at 1732503290706Flushing 1588230740/info: appending metadata at 1732503290729 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732503290729Flushing 1588230740/ns: creating writer at 1732503290734 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732503290752 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732503290752Flushing 1588230740/table: creating writer at 1732503290756 (+4 ms)Flushing 1588230740/table: appending metadata at 1732503290773 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732503290773Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bc3f387: reopening flushed file at 1732503290777 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@86edfe9: reopening flushed file at 1732503290780 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f3d8ce7: reopening flushed file at 1732503290783 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=15, compaction requested=false at 1732503290786 (+3 ms)Writing region close event to WAL at 1732503290788 (+2 ms)Running coprocessor post-close hooks at 1732503290792 (+4 ms)Closed at 1732503290793 (+1 ms) 2024-11-25T02:54:50,793 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T02:54:50,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T02:54:50,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35111-0x1017133be700002, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T02:54:50,849 INFO [RS:1;2c5ea3e41165:35111 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T02:54:50,850 INFO [RS:1;2c5ea3e41165:35111 {}] regionserver.HRegionServer(1031): Exiting; stopping=2c5ea3e41165,35111,1732503285943; zookeeper connection closed. 2024-11-25T02:54:50,850 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1ab72b4a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1ab72b4a 2024-11-25T02:54:50,905 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer(976): stopping server 2c5ea3e41165,34269,1732503285784; all regions closed. 2024-11-25T02:54:50,912 DEBUG [RS:0;2c5ea3e41165:34269 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/oldWALs 2024-11-25T02:54:50,912 INFO [RS:0;2c5ea3e41165:34269 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2c5ea3e41165%2C34269%2C1732503285784.meta:.meta(num 1732503287921) 2024-11-25T02:54:50,914 DEBUG [RS:0;2c5ea3e41165:34269 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/oldWALs 2024-11-25T02:54:50,915 INFO [RS:0;2c5ea3e41165:34269 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2c5ea3e41165%2C34269%2C1732503285784:(num 1732503287481) 2024-11-25T02:54:50,915 DEBUG [RS:0;2c5ea3e41165:34269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:50,915 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T02:54:50,915 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T02:54:50,915 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.ChoreService(370): Chore service for: regionserver/2c5ea3e41165:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T02:54:50,915 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T02:54:50,915 INFO [regionserver/2c5ea3e41165:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T02:54:50,915 INFO [RS:0;2c5ea3e41165:34269 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34269 2024-11-25T02:54:50,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2c5ea3e41165,34269,1732503285784 2024-11-25T02:54:50,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T02:54:50,930 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T02:54:50,940 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2c5ea3e41165,34269,1732503285784] 2024-11-25T02:54:50,948 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2c5ea3e41165,34269,1732503285784 already deleted, retry=false 2024-11-25T02:54:50,948 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2c5ea3e41165,34269,1732503285784 expired; onlineServers=0 2024-11-25T02:54:50,948 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2c5ea3e41165,37837,1732503284943' ***** 2024-11-25T02:54:50,948 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T02:54:50,949 INFO [M:0;2c5ea3e41165:37837 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T02:54:50,949 INFO [M:0;2c5ea3e41165:37837 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T02:54:50,949 DEBUG [M:0;2c5ea3e41165:37837 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T02:54:50,949 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T02:54:50,949 DEBUG [M:0;2c5ea3e41165:37837 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T02:54:50,949 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster-HFileCleaner.small.0-1732503287110 {}] cleaner.HFileCleaner(306): Exit Thread[master/2c5ea3e41165:0:becomeActiveMaster-HFileCleaner.small.0-1732503287110,5,FailOnTimeoutGroup] 2024-11-25T02:54:50,949 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster-HFileCleaner.large.0-1732503287109 {}] cleaner.HFileCleaner(306): Exit Thread[master/2c5ea3e41165:0:becomeActiveMaster-HFileCleaner.large.0-1732503287109,5,FailOnTimeoutGroup] 2024-11-25T02:54:50,949 INFO [M:0;2c5ea3e41165:37837 {}] hbase.ChoreService(370): Chore service for: master/2c5ea3e41165:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T02:54:50,950 INFO [M:0;2c5ea3e41165:37837 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T02:54:50,950 DEBUG [M:0;2c5ea3e41165:37837 {}] master.HMaster(1795): Stopping service threads 2024-11-25T02:54:50,950 INFO [M:0;2c5ea3e41165:37837 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T02:54:50,950 INFO [M:0;2c5ea3e41165:37837 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T02:54:50,950 INFO [M:0;2c5ea3e41165:37837 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T02:54:50,950 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T02:54:50,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T02:54:50,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:50,957 DEBUG [M:0;2c5ea3e41165:37837 {}] zookeeper.ZKUtil(347): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T02:54:50,957 WARN [M:0;2c5ea3e41165:37837 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T02:54:50,957 INFO [M:0;2c5ea3e41165:37837 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/.lastflushedseqids 2024-11-25T02:54:50,967 INFO [M:0;2c5ea3e41165:37837 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T02:54:50,967 INFO [M:0;2c5ea3e41165:37837 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T02:54:50,967 DEBUG [M:0;2c5ea3e41165:37837 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T02:54:50,968 INFO [M:0;2c5ea3e41165:37837 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T02:54:50,968 DEBUG [M:0;2c5ea3e41165:37837 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T02:54:50,968 DEBUG [M:0;2c5ea3e41165:37837 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T02:54:50,968 DEBUG [M:0;2c5ea3e41165:37837 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T02:54:50,968 INFO [M:0;2c5ea3e41165:37837 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-25T02:54:50,976 DEBUG [M:0;2c5ea3e41165:37837 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ba5db7ab2b124c7fa47cfa7b0f7e0b62 2024-11-25T02:54:50,988 DEBUG [M:0;2c5ea3e41165:37837 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ba5db7ab2b124c7fa47cfa7b0f7e0b62 is 82, key is hbase:meta,,1/info:regioninfo/1732503287981/Put/seqid=0 2024-11-25T02:54:50,989 INFO [M:0;2c5ea3e41165:37837 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ba5db7ab2b124c7fa47cfa7b0f7e0b62 2024-11-25T02:54:50,996 DEBUG [M:0;2c5ea3e41165:37837 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 05980dd53cbc453b95d4f02e2b20c333 2024-11-25T02:54:51,009 DEBUG [M:0;2c5ea3e41165:37837 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/05980dd53cbc453b95d4f02e2b20c333 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732503288696/Put/seqid=0 2024-11-25T02:54:51,010 INFO [M:0;2c5ea3e41165:37837 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/05980dd53cbc453b95d4f02e2b20c333 2024-11-25T02:54:51,016 DEBUG [M:0;2c5ea3e41165:37837 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 23d46dba703b446daa5d24aa95c020bb 2024-11-25T02:54:51,028 DEBUG [M:0;2c5ea3e41165:37837 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/23d46dba703b446daa5d24aa95c020bb is 69, key is 2c5ea3e41165,34269,1732503285784/rs:state/1732503287163/Put/seqid=0 2024-11-25T02:54:51,029 INFO [M:0;2c5ea3e41165:37837 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/23d46dba703b446daa5d24aa95c020bb 2024-11-25T02:54:51,031 DEBUG [M:0;2c5ea3e41165:37837 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ba5db7ab2b124c7fa47cfa7b0f7e0b62 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ba5db7ab2b124c7fa47cfa7b0f7e0b62 2024-11-25T02:54:51,032 INFO [M:0;2c5ea3e41165:37837 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ba5db7ab2b124c7fa47cfa7b0f7e0b62, entries=8, sequenceid=84, filesize=5.5 K 2024-11-25T02:54:51,033 DEBUG [M:0;2c5ea3e41165:37837 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/05980dd53cbc453b95d4f02e2b20c333 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/05980dd53cbc453b95d4f02e2b20c333 2024-11-25T02:54:51,034 INFO [M:0;2c5ea3e41165:37837 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/05980dd53cbc453b95d4f02e2b20c333, entries=9, sequenceid=84, filesize=6.5 K 2024-11-25T02:54:51,034 DEBUG [M:0;2c5ea3e41165:37837 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/23d46dba703b446daa5d24aa95c020bb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/23d46dba703b446daa5d24aa95c020bb 2024-11-25T02:54:51,036 INFO [M:0;2c5ea3e41165:37837 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/23d46dba703b446daa5d24aa95c020bb, entries=2, sequenceid=84, filesize=5.1 K 2024-11-25T02:54:51,036 INFO [M:0;2c5ea3e41165:37837 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=84, compaction requested=false 2024-11-25T02:54:51,038 INFO [M:0;2c5ea3e41165:37837 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T02:54:51,039 DEBUG [M:0;2c5ea3e41165:37837 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732503290967Disabling compacts and flushes for region at 1732503290967Disabling writes for close at 1732503290968 (+1 ms)Obtaining lock to block concurrent updates at 1732503290968Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732503290968Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732503290968Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732503290969 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732503290969Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732503290988 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732503290988Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732503290991 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732503291008 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732503291008Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732503291011 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732503291028 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732503291028Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a8d19f4: reopening flushed file at 1732503291030 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aa0f3b9: reopening flushed file at 1732503291032 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c4e45f9: reopening flushed file at 1732503291034 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=84, compaction requested=false at 1732503291037 (+3 ms)Writing region close event to WAL at 1732503291038 (+1 ms)Closed at 1732503291038 2024-11-25T02:54:51,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T02:54:51,040 INFO [RS:0;2c5ea3e41165:34269 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T02:54:51,040 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T02:54:51,040 INFO [M:0;2c5ea3e41165:37837 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T02:54:51,040 INFO [RS:0;2c5ea3e41165:34269 {}] regionserver.HRegionServer(1031): Exiting; stopping=2c5ea3e41165,34269,1732503285784; zookeeper connection closed. 2024-11-25T02:54:51,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34269-0x1017133be700001, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T02:54:51,040 INFO [M:0;2c5ea3e41165:37837 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37837 2024-11-25T02:54:51,040 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2b09362b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2b09362b 2024-11-25T02:54:51,041 INFO [M:0;2c5ea3e41165:37837 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T02:54:51,041 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-25T02:54:51,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T02:54:51,165 INFO [M:0;2c5ea3e41165:37837 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T02:54:51,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37837-0x1017133be700000, quorum=127.0.0.1:63189, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T02:54:51,173 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T02:54:51,194 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-25T02:54:51,194 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:51,201 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732503290778 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732503290781 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/2c5ea3e41165:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: hfile-prefetch-1732503289066 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/2c5ea3e41165:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1732503290615 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/2c5ea3e41165:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/2c5ea3e41165:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=398 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=12132 (was 12501) 2024-11-25T02:54:51,207 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=398, ProcessCount=11, AvailableMemoryMB=12132 2024-11-25T02:54:51,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/hadoop-log-dir so I do NOT create it in target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb 2024-11-25T02:54:51,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/hadoop-tmp-dir so I do NOT create it in target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb 2024-11-25T02:54:51,207 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:51,210 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/cluster_4b4c2c3e-52b7-8e0b-99ff-af09efefa14a, deleteOnExit=true 2024-11-25T02:54:51,212 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/cluster_4b4c2c3e-52b7-8e0b-99ff-af09efefa14a/zookeeper_0, clientPort=51409, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/cluster_4b4c2c3e-52b7-8e0b-99ff-af09efefa14a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/cluster_4b4c2c3e-52b7-8e0b-99ff-af09efefa14a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T02:54:51,213 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51409 2024-11-25T02:54:51,223 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a with version=8 2024-11-25T02:54:51,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/hbase-staging 2024-11-25T02:54:51,226 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2c5ea3e41165:0 server-side Connection retries=45 2024-11-25T02:54:51,226 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:51,227 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:51,227 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T02:54:51,227 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:51,227 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T02:54:51,227 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T02:54:51,227 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T02:54:51,228 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43523 2024-11-25T02:54:51,230 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43523 connecting to ZooKeeper ensemble=127.0.0.1:51409 2024-11-25T02:54:51,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:435230x0, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T02:54:51,274 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43523-0x1017133d8110000 connected 2024-11-25T02:54:51,349 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:51,349 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:51,352 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T02:54:51,352 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a, hbase.cluster.distributed=false 2024-11-25T02:54:51,356 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T02:54:51,358 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43523 2024-11-25T02:54:51,358 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43523 2024-11-25T02:54:51,359 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43523 2024-11-25T02:54:51,360 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43523 2024-11-25T02:54:51,360 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43523 2024-11-25T02:54:51,375 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2c5ea3e41165:0 server-side Connection retries=45 2024-11-25T02:54:51,375 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:51,375 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:51,375 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T02:54:51,375 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:51,375 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T02:54:51,375 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T02:54:51,375 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T02:54:51,376 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34583 2024-11-25T02:54:51,377 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34583 connecting to ZooKeeper ensemble=127.0.0.1:51409 2024-11-25T02:54:51,378 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:51,378 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:51,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:345830x0, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T02:54:51,390 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34583-0x1017133d8110001 connected 2024-11-25T02:54:51,390 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T02:54:51,391 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T02:54:51,392 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/bucket.cache 2024-11-25T02:54:51,392 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T02:54:51,392 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T02:54:51,392 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T02:54:51,392 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T02:54:51,392 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-25T02:54:51,392 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T02:54:51,393 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T02:54:51,393 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T02:54:51,393 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/bucket.persistence is complete. 2024-11-25T02:54:51,393 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T02:54:51,395 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T02:54:51,395 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34583 2024-11-25T02:54:51,395 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34583 2024-11-25T02:54:51,398 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34583 2024-11-25T02:54:51,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34583 2024-11-25T02:54:51,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34583 2024-11-25T02:54:51,418 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2c5ea3e41165:0 server-side Connection retries=45 2024-11-25T02:54:51,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:51,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:51,419 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T02:54:51,419 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:51,419 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T02:54:51,419 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T02:54:51,419 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T02:54:51,420 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44817 2024-11-25T02:54:51,422 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44817 connecting to ZooKeeper ensemble=127.0.0.1:51409 2024-11-25T02:54:51,422 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:51,422 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:51,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:448170x0, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T02:54:51,432 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44817-0x1017133d8110002 connected 2024-11-25T02:54:51,432 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T02:54:51,432 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T02:54:51,433 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/bucket.cache 2024-11-25T02:54:51,433 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T02:54:51,433 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T02:54:51,433 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T02:54:51,433 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T02:54:51,434 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T02:54:51,434 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-25T02:54:51,434 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T02:54:51,434 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/bucket.persistence is complete. 2024-11-25T02:54:51,434 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T02:54:51,435 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T02:54:51,437 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T02:54:51,437 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44817 2024-11-25T02:54:51,437 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44817 2024-11-25T02:54:51,438 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44817 2024-11-25T02:54:51,439 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44817 2024-11-25T02:54:51,439 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44817 2024-11-25T02:54:51,453 DEBUG [M:0;2c5ea3e41165:43523 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2c5ea3e41165:43523 2024-11-25T02:54:51,453 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2c5ea3e41165,43523,1732503291226 2024-11-25T02:54:51,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T02:54:51,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T02:54:51,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T02:54:51,465 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2c5ea3e41165,43523,1732503291226 2024-11-25T02:54:51,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T02:54:51,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T02:54:51,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:51,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:51,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:51,474 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T02:54:51,475 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2c5ea3e41165,43523,1732503291226 from backup master directory 2024-11-25T02:54:51,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T02:54:51,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2c5ea3e41165,43523,1732503291226 2024-11-25T02:54:51,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T02:54:51,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T02:54:51,482 WARN [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T02:54:51,482 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2c5ea3e41165,43523,1732503291226 2024-11-25T02:54:51,484 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/hbase.id] with ID: 45c415a0-0ab9-4b81-9f1c-83473716c96b 2024-11-25T02:54:51,484 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/.tmp/hbase.id 2024-11-25T02:54:51,488 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/hbase.id] 2024-11-25T02:54:51,531 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:51,532 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T02:54:51,535 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-25T02:54:51,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:51,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:51,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:51,560 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T02:54:51,561 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T02:54:51,561 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T02:54:51,585 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store 2024-11-25T02:54:51,594 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T02:54:51,594 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T02:54:51,594 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T02:54:51,594 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T02:54:51,595 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T02:54:51,595 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T02:54:51,595 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T02:54:51,595 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732503291594Disabling compacts and flushes for region at 1732503291594Disabling writes for close at 1732503291595 (+1 ms)Writing region close event to WAL at 1732503291595Closed at 1732503291595 2024-11-25T02:54:51,595 WARN [master/2c5ea3e41165:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/.initializing 2024-11-25T02:54:51,595 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/WALs/2c5ea3e41165,43523,1732503291226 2024-11-25T02:54:51,601 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T02:54:51,604 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2c5ea3e41165%2C43523%2C1732503291226, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/WALs/2c5ea3e41165,43523,1732503291226, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/oldWALs, maxLogs=10 2024-11-25T02:54:51,620 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/WALs/2c5ea3e41165,43523,1732503291226/2c5ea3e41165%2C43523%2C1732503291226.1732503291605 2024-11-25T02:54:51,621 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T02:54:51,621 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T02:54:51,621 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T02:54:51,621 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:51,621 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:51,622 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:51,627 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T02:54:51,627 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:51,627 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:51,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:51,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T02:54:51,632 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:51,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T02:54:51,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:51,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T02:54:51,636 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:51,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T02:54:51,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:51,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T02:54:51,639 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:51,640 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T02:54:51,640 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:51,640 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:51,640 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:51,641 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:51,641 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:51,641 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T02:54:51,641 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T02:54:51,646 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T02:54:51,647 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59878183, jitterRate=-0.10774554312229156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T02:54:51,648 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732503291621Initializing all the Stores at 1732503291621Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732503291622 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732503291622Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732503291622Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732503291622Cleaning up temporary data from old regions at 1732503291641 (+19 ms)Region opened successfully at 1732503291648 (+7 ms) 2024-11-25T02:54:51,649 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T02:54:51,653 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32afb2e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2c5ea3e41165/172.17.0.2:0 2024-11-25T02:54:51,654 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T02:54:51,654 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T02:54:51,654 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T02:54:51,654 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T02:54:51,655 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T02:54:51,655 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-25T02:54:51,655 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T02:54:51,657 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T02:54:51,658 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T02:54:51,678 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T02:54:51,679 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T02:54:51,680 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T02:54:51,689 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T02:54:51,690 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T02:54:51,691 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T02:54:51,698 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T02:54:51,699 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T02:54:51,706 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T02:54:51,709 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T02:54:51,714 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T02:54:51,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T02:54:51,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T02:54:51,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T02:54:51,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:51,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:51,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:51,724 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2c5ea3e41165,43523,1732503291226, sessionid=0x1017133d8110000, setting cluster-up flag (Was=false) 2024-11-25T02:54:51,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:51,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:51,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:51,765 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T02:54:51,766 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2c5ea3e41165,43523,1732503291226 2024-11-25T02:54:51,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:51,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:51,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:51,806 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T02:54:51,808 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2c5ea3e41165,43523,1732503291226 2024-11-25T02:54:51,809 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T02:54:51,810 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T02:54:51,811 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T02:54:51,811 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T02:54:51,811 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2c5ea3e41165,43523,1732503291226 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T02:54:51,813 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2c5ea3e41165:0, corePoolSize=5, maxPoolSize=5 2024-11-25T02:54:51,814 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2c5ea3e41165:0, corePoolSize=5, maxPoolSize=5 2024-11-25T02:54:51,814 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2c5ea3e41165:0, corePoolSize=5, maxPoolSize=5 2024-11-25T02:54:51,814 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2c5ea3e41165:0, corePoolSize=5, maxPoolSize=5 2024-11-25T02:54:51,814 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2c5ea3e41165:0, corePoolSize=10, maxPoolSize=10 2024-11-25T02:54:51,814 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,814 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2c5ea3e41165:0, corePoolSize=2, maxPoolSize=2 2024-11-25T02:54:51,814 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,818 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T02:54:51,818 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T02:54:51,818 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:51,819 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T02:54:51,823 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732503321822 2024-11-25T02:54:51,823 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T02:54:51,823 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T02:54:51,823 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T02:54:51,823 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T02:54:51,823 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T02:54:51,823 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T02:54:51,826 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,827 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T02:54:51,827 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T02:54:51,827 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T02:54:51,831 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T02:54:51,831 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T02:54:51,831 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2c5ea3e41165:0:becomeActiveMaster-HFileCleaner.large.0-1732503291831,5,FailOnTimeoutGroup] 2024-11-25T02:54:51,831 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2c5ea3e41165:0:becomeActiveMaster-HFileCleaner.small.0-1732503291831,5,FailOnTimeoutGroup] 2024-11-25T02:54:51,831 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,831 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T02:54:51,831 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,832 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,841 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T02:54:51,842 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a 2024-11-25T02:54:51,844 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer(746): ClusterId : 45c415a0-0ab9-4b81-9f1c-83473716c96b 2024-11-25T02:54:51,844 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.HRegionServer(746): ClusterId : 45c415a0-0ab9-4b81-9f1c-83473716c96b 2024-11-25T02:54:51,844 DEBUG [RS:0;2c5ea3e41165:34583 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T02:54:51,844 DEBUG [RS:1;2c5ea3e41165:44817 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T02:54:51,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T02:54:51,854 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T02:54:51,857 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T02:54:51,857 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:51,858 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:51,858 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T02:54:51,861 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T02:54:51,861 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:51,861 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:51,861 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T02:54:51,862 DEBUG [RS:1;2c5ea3e41165:44817 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T02:54:51,862 DEBUG [RS:1;2c5ea3e41165:44817 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T02:54:51,863 DEBUG [RS:0;2c5ea3e41165:34583 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T02:54:51,863 DEBUG [RS:0;2c5ea3e41165:34583 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T02:54:51,864 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T02:54:51,865 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:51,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:51,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T02:54:51,868 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T02:54:51,868 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:51,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:51,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T02:54:51,869 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740 2024-11-25T02:54:51,869 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740 2024-11-25T02:54:51,869 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T02:54:51,869 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T02:54:51,869 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T02:54:51,870 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T02:54:51,875 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T02:54:51,875 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71517825, jitterRate=0.06569863855838776}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T02:54:51,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732503291853Initializing all the Stores at 1732503291854 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732503291854Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732503291854Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732503291854Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732503291854Cleaning up temporary data from old regions at 1732503291869 (+15 ms)Region opened successfully at 1732503291876 (+7 ms) 2024-11-25T02:54:51,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T02:54:51,876 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T02:54:51,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T02:54:51,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T02:54:51,876 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T02:54:51,877 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T02:54:51,877 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732503291876Disabling compacts and flushes for region at 1732503291876Disabling writes for close at 1732503291876Writing region close event to WAL at 1732503291877 (+1 ms)Closed at 1732503291877 2024-11-25T02:54:51,878 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T02:54:51,878 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T02:54:51,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T02:54:51,879 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T02:54:51,880 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T02:54:51,882 DEBUG [RS:1;2c5ea3e41165:44817 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T02:54:51,882 DEBUG [RS:0;2c5ea3e41165:34583 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T02:54:51,883 DEBUG [RS:0;2c5ea3e41165:34583 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f4f8e15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2c5ea3e41165/172.17.0.2:0 2024-11-25T02:54:51,883 DEBUG [RS:1;2c5ea3e41165:44817 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ea26e4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2c5ea3e41165/172.17.0.2:0 2024-11-25T02:54:51,899 DEBUG [RS:1;2c5ea3e41165:44817 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2c5ea3e41165:44817 2024-11-25T02:54:51,899 DEBUG [RS:0;2c5ea3e41165:34583 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2c5ea3e41165:34583 2024-11-25T02:54:51,900 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T02:54:51,900 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T02:54:51,900 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T02:54:51,900 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T02:54:51,900 DEBUG [RS:1;2c5ea3e41165:44817 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T02:54:51,900 DEBUG [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T02:54:51,901 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer(2659): reportForDuty to master=2c5ea3e41165,43523,1732503291226 with port=34583, startcode=1732503291374 2024-11-25T02:54:51,901 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.HRegionServer(2659): reportForDuty to master=2c5ea3e41165,43523,1732503291226 with port=44817, startcode=1732503291418 2024-11-25T02:54:51,902 DEBUG [RS:0;2c5ea3e41165:34583 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T02:54:51,902 DEBUG [RS:1;2c5ea3e41165:44817 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T02:54:51,907 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36169, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T02:54:51,908 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43523 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2c5ea3e41165,34583,1732503291374 2024-11-25T02:54:51,908 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43523 {}] master.ServerManager(517): Registering regionserver=2c5ea3e41165,34583,1732503291374 2024-11-25T02:54:51,910 DEBUG [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a 2024-11-25T02:54:51,910 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35959, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T02:54:51,910 DEBUG [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T02:54:51,910 DEBUG [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T02:54:51,911 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43523 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2c5ea3e41165,44817,1732503291418 2024-11-25T02:54:51,911 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43523 {}] master.ServerManager(517): Registering regionserver=2c5ea3e41165,44817,1732503291418 2024-11-25T02:54:51,912 DEBUG [RS:1;2c5ea3e41165:44817 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a 2024-11-25T02:54:51,913 DEBUG [RS:1;2c5ea3e41165:44817 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T02:54:51,913 DEBUG [RS:1;2c5ea3e41165:44817 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T02:54:51,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T02:54:51,957 DEBUG [RS:0;2c5ea3e41165:34583 {}] zookeeper.ZKUtil(111): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2c5ea3e41165,34583,1732503291374 2024-11-25T02:54:51,957 WARN [RS:0;2c5ea3e41165:34583 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T02:54:51,957 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2c5ea3e41165,34583,1732503291374] 2024-11-25T02:54:51,957 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2c5ea3e41165,44817,1732503291418] 2024-11-25T02:54:51,957 INFO [RS:0;2c5ea3e41165:34583 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T02:54:51,957 DEBUG [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,34583,1732503291374 2024-11-25T02:54:51,957 DEBUG [RS:1;2c5ea3e41165:44817 {}] zookeeper.ZKUtil(111): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2c5ea3e41165,44817,1732503291418 2024-11-25T02:54:51,957 WARN [RS:1;2c5ea3e41165:44817 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T02:54:51,957 INFO [RS:1;2c5ea3e41165:44817 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T02:54:51,957 DEBUG [RS:1;2c5ea3e41165:44817 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,44817,1732503291418 2024-11-25T02:54:51,961 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T02:54:51,961 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T02:54:51,964 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T02:54:51,964 INFO [RS:1;2c5ea3e41165:44817 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T02:54:51,965 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,965 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T02:54:51,965 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T02:54:51,966 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,966 DEBUG [RS:1;2c5ea3e41165:44817 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,966 DEBUG [RS:1;2c5ea3e41165:44817 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,966 DEBUG [RS:1;2c5ea3e41165:44817 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,966 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T02:54:51,966 DEBUG [RS:1;2c5ea3e41165:44817 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,966 DEBUG [RS:1;2c5ea3e41165:44817 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,966 DEBUG [RS:1;2c5ea3e41165:44817 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2c5ea3e41165:0, corePoolSize=2, maxPoolSize=2 2024-11-25T02:54:51,967 DEBUG [RS:1;2c5ea3e41165:44817 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,967 INFO [RS:0;2c5ea3e41165:34583 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T02:54:51,967 DEBUG [RS:1;2c5ea3e41165:44817 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,967 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,967 DEBUG [RS:1;2c5ea3e41165:44817 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,967 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T02:54:51,967 DEBUG [RS:1;2c5ea3e41165:44817 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,967 DEBUG [RS:1;2c5ea3e41165:44817 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,967 DEBUG [RS:1;2c5ea3e41165:44817 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,967 DEBUG [RS:1;2c5ea3e41165:44817 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2c5ea3e41165:0, corePoolSize=3, maxPoolSize=3 2024-11-25T02:54:51,967 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T02:54:51,967 DEBUG [RS:1;2c5ea3e41165:44817 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2c5ea3e41165:0, corePoolSize=3, maxPoolSize=3 2024-11-25T02:54:51,967 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,968 DEBUG [RS:0;2c5ea3e41165:34583 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,968 DEBUG [RS:0;2c5ea3e41165:34583 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,968 DEBUG [RS:0;2c5ea3e41165:34583 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,968 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,968 DEBUG [RS:0;2c5ea3e41165:34583 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,968 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,968 DEBUG [RS:0;2c5ea3e41165:34583 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,968 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,968 DEBUG [RS:0;2c5ea3e41165:34583 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2c5ea3e41165:0, corePoolSize=2, maxPoolSize=2 2024-11-25T02:54:51,968 DEBUG [RS:0;2c5ea3e41165:34583 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,968 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,968 DEBUG [RS:0;2c5ea3e41165:34583 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,968 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,968 DEBUG [RS:0;2c5ea3e41165:34583 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,968 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,44817,1732503291418-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T02:54:51,968 DEBUG [RS:0;2c5ea3e41165:34583 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,969 DEBUG [RS:0;2c5ea3e41165:34583 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,969 DEBUG [RS:0;2c5ea3e41165:34583 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:51,969 DEBUG [RS:0;2c5ea3e41165:34583 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2c5ea3e41165:0, corePoolSize=3, maxPoolSize=3 2024-11-25T02:54:51,969 DEBUG [RS:0;2c5ea3e41165:34583 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2c5ea3e41165:0, corePoolSize=3, maxPoolSize=3 2024-11-25T02:54:51,972 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,972 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,972 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,972 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,973 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,973 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,34583,1732503291374-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T02:54:51,986 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T02:54:51,987 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,34583,1732503291374-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,987 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,987 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.Replication(171): 2c5ea3e41165,34583,1732503291374 started 2024-11-25T02:54:51,991 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T02:54:51,991 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,44817,1732503291418-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,992 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:51,992 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.Replication(171): 2c5ea3e41165,44817,1732503291418 started 2024-11-25T02:54:52,002 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:52,002 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer(1482): Serving as 2c5ea3e41165,34583,1732503291374, RpcServer on 2c5ea3e41165/172.17.0.2:34583, sessionid=0x1017133d8110001 2024-11-25T02:54:52,002 DEBUG [RS:0;2c5ea3e41165:34583 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T02:54:52,002 DEBUG [RS:0;2c5ea3e41165:34583 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2c5ea3e41165,34583,1732503291374 2024-11-25T02:54:52,002 DEBUG [RS:0;2c5ea3e41165:34583 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2c5ea3e41165,34583,1732503291374' 2024-11-25T02:54:52,002 DEBUG [RS:0;2c5ea3e41165:34583 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T02:54:52,003 DEBUG [RS:0;2c5ea3e41165:34583 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T02:54:52,004 DEBUG [RS:0;2c5ea3e41165:34583 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T02:54:52,004 DEBUG [RS:0;2c5ea3e41165:34583 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T02:54:52,004 DEBUG [RS:0;2c5ea3e41165:34583 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2c5ea3e41165,34583,1732503291374 2024-11-25T02:54:52,004 DEBUG [RS:0;2c5ea3e41165:34583 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2c5ea3e41165,34583,1732503291374' 2024-11-25T02:54:52,004 DEBUG [RS:0;2c5ea3e41165:34583 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T02:54:52,004 DEBUG [RS:0;2c5ea3e41165:34583 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T02:54:52,005 DEBUG [RS:0;2c5ea3e41165:34583 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T02:54:52,005 INFO [RS:0;2c5ea3e41165:34583 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T02:54:52,005 INFO [RS:0;2c5ea3e41165:34583 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T02:54:52,011 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:52,012 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.HRegionServer(1482): Serving as 2c5ea3e41165,44817,1732503291418, RpcServer on 2c5ea3e41165/172.17.0.2:44817, sessionid=0x1017133d8110002 2024-11-25T02:54:52,012 DEBUG [RS:1;2c5ea3e41165:44817 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T02:54:52,012 DEBUG [RS:1;2c5ea3e41165:44817 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2c5ea3e41165,44817,1732503291418 2024-11-25T02:54:52,012 DEBUG [RS:1;2c5ea3e41165:44817 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2c5ea3e41165,44817,1732503291418' 2024-11-25T02:54:52,012 DEBUG [RS:1;2c5ea3e41165:44817 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T02:54:52,013 DEBUG [RS:1;2c5ea3e41165:44817 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T02:54:52,013 DEBUG [RS:1;2c5ea3e41165:44817 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T02:54:52,013 DEBUG [RS:1;2c5ea3e41165:44817 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T02:54:52,013 DEBUG [RS:1;2c5ea3e41165:44817 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2c5ea3e41165,44817,1732503291418 2024-11-25T02:54:52,013 DEBUG [RS:1;2c5ea3e41165:44817 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2c5ea3e41165,44817,1732503291418' 2024-11-25T02:54:52,013 DEBUG [RS:1;2c5ea3e41165:44817 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T02:54:52,014 DEBUG [RS:1;2c5ea3e41165:44817 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T02:54:52,014 DEBUG [RS:1;2c5ea3e41165:44817 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T02:54:52,014 INFO [RS:1;2c5ea3e41165:44817 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T02:54:52,014 INFO [RS:1;2c5ea3e41165:44817 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T02:54:52,030 WARN [2c5ea3e41165:43523 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T02:54:52,106 INFO [RS:0;2c5ea3e41165:34583 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T02:54:52,107 INFO [RS:0;2c5ea3e41165:34583 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2c5ea3e41165%2C34583%2C1732503291374, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,34583,1732503291374, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/oldWALs, maxLogs=55 2024-11-25T02:54:52,115 INFO [RS:1;2c5ea3e41165:44817 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T02:54:52,116 INFO [RS:1;2c5ea3e41165:44817 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2c5ea3e41165%2C44817%2C1732503291418, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,44817,1732503291418, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/oldWALs, maxLogs=55 2024-11-25T02:54:52,129 INFO [RS:0;2c5ea3e41165:34583 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,34583,1732503291374/2c5ea3e41165%2C34583%2C1732503291374.1732503292110 2024-11-25T02:54:52,130 DEBUG [RS:0;2c5ea3e41165:34583 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T02:54:52,134 INFO [RS:1;2c5ea3e41165:44817 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,44817,1732503291418/2c5ea3e41165%2C44817%2C1732503291418.1732503292118 2024-11-25T02:54:52,134 DEBUG [RS:1;2c5ea3e41165:44817 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T02:54:52,281 DEBUG [2c5ea3e41165:43523 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-25T02:54:52,281 DEBUG [2c5ea3e41165:43523 {}] balancer.BalancerClusterState(204): Hosts are {2c5ea3e41165=0} racks are {/default-rack=0} 2024-11-25T02:54:52,282 DEBUG [2c5ea3e41165:43523 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T02:54:52,282 DEBUG [2c5ea3e41165:43523 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T02:54:52,282 DEBUG [2c5ea3e41165:43523 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T02:54:52,282 DEBUG [2c5ea3e41165:43523 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T02:54:52,282 INFO [2c5ea3e41165:43523 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T02:54:52,282 INFO [2c5ea3e41165:43523 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T02:54:52,282 DEBUG [2c5ea3e41165:43523 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T02:54:52,283 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2c5ea3e41165,34583,1732503291374 2024-11-25T02:54:52,284 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2c5ea3e41165,34583,1732503291374, state=OPENING 2024-11-25T02:54:52,312 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T02:54:52,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:52,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:52,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:52,323 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T02:54:52,323 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T02:54:52,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2c5ea3e41165,34583,1732503291374}] 2024-11-25T02:54:52,324 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T02:54:52,324 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T02:54:52,476 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T02:54:52,478 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47363, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T02:54:52,480 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T02:54:52,480 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T02:54:52,481 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T02:54:52,481 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2c5ea3e41165%2C34583%2C1732503291374.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,34583,1732503291374, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/oldWALs, maxLogs=55 2024-11-25T02:54:52,498 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,34583,1732503291374/2c5ea3e41165%2C34583%2C1732503291374.meta.1732503292482.meta 2024-11-25T02:54:52,499 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T02:54:52,499 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T02:54:52,499 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T02:54:52,499 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T02:54:52,499 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T02:54:52,500 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T02:54:52,500 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T02:54:52,500 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T02:54:52,500 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T02:54:52,501 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T02:54:52,502 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T02:54:52,502 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:52,502 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:52,502 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T02:54:52,503 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T02:54:52,503 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:52,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:52,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T02:54:52,504 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T02:54:52,504 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:52,504 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:52,504 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T02:54:52,505 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T02:54:52,505 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:52,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T02:54:52,506 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T02:54:52,506 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740 2024-11-25T02:54:52,507 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740 2024-11-25T02:54:52,507 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T02:54:52,507 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T02:54:52,507 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T02:54:52,508 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T02:54:52,508 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60722463, jitterRate=-0.09516479074954987}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T02:54:52,509 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T02:54:52,509 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732503292500Writing region info on filesystem at 1732503292500Initializing all the Stores at 1732503292500Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732503292500Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732503292501 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732503292501Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732503292501Cleaning up temporary data from old regions at 1732503292507 (+6 ms)Running coprocessor post-open hooks at 1732503292509 (+2 ms)Region opened successfully at 1732503292509 2024-11-25T02:54:52,510 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732503292476 2024-11-25T02:54:52,513 DEBUG [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T02:54:52,513 INFO [RS_OPEN_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T02:54:52,513 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2c5ea3e41165,34583,1732503291374 2024-11-25T02:54:52,514 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2c5ea3e41165,34583,1732503291374, state=OPEN 2024-11-25T02:54:52,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T02:54:52,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T02:54:52,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T02:54:52,545 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2c5ea3e41165,34583,1732503291374 2024-11-25T02:54:52,546 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T02:54:52,546 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T02:54:52,546 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T02:54:52,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T02:54:52,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2c5ea3e41165,34583,1732503291374 in 222 msec 2024-11-25T02:54:52,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T02:54:52,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 672 msec 2024-11-25T02:54:52,555 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T02:54:52,555 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T02:54:52,557 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T02:54:52,557 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2c5ea3e41165,34583,1732503291374, seqNum=-1] 2024-11-25T02:54:52,558 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T02:54:52,560 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52611, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T02:54:52,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 755 msec 2024-11-25T02:54:52,565 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732503292565, completionTime=-1 2024-11-25T02:54:52,566 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-25T02:54:52,566 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T02:54:52,568 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T02:54:52,568 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732503352568 2024-11-25T02:54:52,569 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732503412568 2024-11-25T02:54:52,569 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-25T02:54:52,569 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-25T02:54:52,570 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,43523,1732503291226-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:52,570 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,43523,1732503291226-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:52,570 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,43523,1732503291226-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:52,570 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2c5ea3e41165:43523, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:52,570 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:52,570 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:52,573 DEBUG [master/2c5ea3e41165:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T02:54:52,576 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.094sec 2024-11-25T02:54:52,577 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T02:54:52,577 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T02:54:52,577 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T02:54:52,577 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T02:54:52,577 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T02:54:52,577 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,43523,1732503291226-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T02:54:52,577 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,43523,1732503291226-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T02:54:52,578 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T02:54:52,578 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T02:54:52,578 INFO [master/2c5ea3e41165:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,43523,1732503291226-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:52,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22568a62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T02:54:52,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2c5ea3e41165,43523,-1 for getting cluster id 2024-11-25T02:54:52,646 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T02:54:52,648 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45c415a0-0ab9-4b81-9f1c-83473716c96b' 2024-11-25T02:54:52,649 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T02:54:52,650 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45c415a0-0ab9-4b81-9f1c-83473716c96b" 2024-11-25T02:54:52,650 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52b4b21e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T02:54:52,651 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2c5ea3e41165,43523,-1] 2024-11-25T02:54:52,651 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T02:54:52,652 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:52,656 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T02:54:52,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51acda6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T02:54:52,659 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T02:54:52,660 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2c5ea3e41165,34583,1732503291374, seqNum=-1] 2024-11-25T02:54:52,661 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T02:54:52,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T02:54:52,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2c5ea3e41165,43523,1732503291226 2024-11-25T02:54:52,667 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T02:54:52,668 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 2c5ea3e41165,43523,1732503291226 2024-11-25T02:54:52,668 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7a3410cf 2024-11-25T02:54:52,668 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T02:54:52,670 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T02:54:52,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43523 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732503292666', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T02:54:52,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43523 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732503292666 2024-11-25T02:54:52,673 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732503292666 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T02:54:52,674 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:52,674 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43523 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732503292666" procId is: 4 2024-11-25T02:54:52,675 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732503292666 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T02:54:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T02:54:52,684 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732503292666-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1b5a6ce08ca401de35fefffe6c3da2ee, NAME => 'testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732503292666', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a 2024-11-25T02:54:52,691 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732503292666-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T02:54:52,691 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732503292666-pool-0 {}] regionserver.HRegion(1722): Closing 1b5a6ce08ca401de35fefffe6c3da2ee, disabling compactions & flushes 2024-11-25T02:54:52,691 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732503292666-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:52,691 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732503292666-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:52,691 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732503292666-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. after waiting 0 ms 2024-11-25T02:54:52,691 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732503292666-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:52,691 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732503292666-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:52,691 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732503292666-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1b5a6ce08ca401de35fefffe6c3da2ee: Waiting for close lock at 1732503292691Disabling compacts and flushes for region at 1732503292691Disabling writes for close at 1732503292691Writing region close event to WAL at 1732503292691Closed at 1732503292691 2024-11-25T02:54:52,693 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732503292666 execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T02:54:52,693 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732503292693"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732503292693"}]},"ts":"1732503292693"} 2024-11-25T02:54:52,696 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T02:54:52,697 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732503292666 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T02:54:52,697 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732503292666","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732503292697"}]},"ts":"1732503292697"} 2024-11-25T02:54:52,700 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732503292666, state=ENABLING in hbase:meta 2024-11-25T02:54:52,700 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {2c5ea3e41165=0} racks are {/default-rack=0} 2024-11-25T02:54:52,701 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T02:54:52,701 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T02:54:52,701 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T02:54:52,701 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T02:54:52,701 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T02:54:52,701 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T02:54:52,701 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T02:54:52,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732503292666, region=1b5a6ce08ca401de35fefffe6c3da2ee, ASSIGN}] 2024-11-25T02:54:52,703 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732503292666, region=1b5a6ce08ca401de35fefffe6c3da2ee, ASSIGN 2024-11-25T02:54:52,703 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732503292666, region=1b5a6ce08ca401de35fefffe6c3da2ee, ASSIGN; state=OFFLINE, location=2c5ea3e41165,44817,1732503291418; forceNewPlan=false, retain=false 2024-11-25T02:54:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T02:54:52,854 INFO [2c5ea3e41165:43523 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T02:54:52,854 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1b5a6ce08ca401de35fefffe6c3da2ee, regionState=OPENING, regionLocation=2c5ea3e41165,44817,1732503291418 2024-11-25T02:54:52,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732503292666, region=1b5a6ce08ca401de35fefffe6c3da2ee, ASSIGN because future has completed 2024-11-25T02:54:52,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b5a6ce08ca401de35fefffe6c3da2ee, server=2c5ea3e41165,44817,1732503291418}] 2024-11-25T02:54:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T02:54:53,012 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T02:54:53,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46739, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T02:54:53,017 INFO [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:53,018 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b5a6ce08ca401de35fefffe6c3da2ee, NAME => 'testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee.', STARTKEY => '', ENDKEY => ''} 2024-11-25T02:54:53,019 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732503292666 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:53,019 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T02:54:53,019 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:53,019 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:53,021 INFO [StoreOpener-1b5a6ce08ca401de35fefffe6c3da2ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:53,025 INFO [StoreOpener-1b5a6ce08ca401de35fefffe6c3da2ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b5a6ce08ca401de35fefffe6c3da2ee columnFamilyName family 2024-11-25T02:54:53,025 DEBUG [StoreOpener-1b5a6ce08ca401de35fefffe6c3da2ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:53,025 INFO [StoreOpener-1b5a6ce08ca401de35fefffe6c3da2ee-1 {}] regionserver.HStore(327): Store=1b5a6ce08ca401de35fefffe6c3da2ee/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T02:54:53,025 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:53,026 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:53,026 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:53,026 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:53,026 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:53,027 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:53,032 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T02:54:53,032 INFO [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1b5a6ce08ca401de35fefffe6c3da2ee; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58762391, jitterRate=-0.12437213957309723}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T02:54:53,032 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:53,033 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1b5a6ce08ca401de35fefffe6c3da2ee: Running coprocessor pre-open hook at 1732503293019Writing region info on filesystem at 1732503293019Initializing all the Stores at 1732503293020 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732503293020Cleaning up temporary data from old regions at 1732503293026 (+6 ms)Running coprocessor post-open hooks at 1732503293032 (+6 ms)Region opened successfully at 1732503293033 (+1 ms) 2024-11-25T02:54:53,034 INFO [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee., pid=6, masterSystemTime=1732503293011 2024-11-25T02:54:53,036 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:53,036 INFO [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:53,036 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1b5a6ce08ca401de35fefffe6c3da2ee, regionState=OPEN, openSeqNum=2, regionLocation=2c5ea3e41165,44817,1732503291418 2024-11-25T02:54:53,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b5a6ce08ca401de35fefffe6c3da2ee, server=2c5ea3e41165,44817,1732503291418 because future has completed 2024-11-25T02:54:53,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T02:54:53,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1b5a6ce08ca401de35fefffe6c3da2ee, server=2c5ea3e41165,44817,1732503291418 in 182 msec 2024-11-25T02:54:53,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T02:54:53,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732503292666, region=1b5a6ce08ca401de35fefffe6c3da2ee, ASSIGN in 341 msec 2024-11-25T02:54:53,044 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732503292666 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T02:54:53,045 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732503292666","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732503293044"}]},"ts":"1732503293044"} 2024-11-25T02:54:53,047 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732503292666, state=ENABLED in hbase:meta 2024-11-25T02:54:53,048 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732503292666 execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T02:54:53,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732503292666 in 377 msec 2024-11-25T02:54:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T02:54:53,308 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732503292666 completed 2024-11-25T02:54:53,308 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732503292666 get assigned. Timeout = 60000ms 2024-11-25T02:54:53,309 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T02:54:53,314 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732503292666 assigned to meta. Checking AM states. 2024-11-25T02:54:53,315 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T02:54:53,315 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732503292666 assigned. 2024-11-25T02:54:53,321 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732503292666', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee., hostname=2c5ea3e41165,44817,1732503291418, seqNum=2] 2024-11-25T02:54:53,322 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T02:54:53,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T02:54:53,329 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1b5a6ce08ca401de35fefffe6c3da2ee 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-25T02:54:53,337 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8f5c75482ae948cc9276033c3fe9de04 2024-11-25T02:54:53,351 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/.tmp/family/8f5c75482ae948cc9276033c3fe9de04 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-25T02:54:53,351 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/.tmp/family/8f5c75482ae948cc9276033c3fe9de04 2024-11-25T02:54:53,354 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/.tmp/family/8f5c75482ae948cc9276033c3fe9de04 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/family/8f5c75482ae948cc9276033c3fe9de04 2024-11-25T02:54:53,355 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/family/8f5c75482ae948cc9276033c3fe9de04, delay=940 ms 2024-11-25T02:54:53,357 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/family/8f5c75482ae948cc9276033c3fe9de04, entries=2, sequenceid=6, filesize=4.9 K 2024-11-25T02:54:53,357 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 1b5a6ce08ca401de35fefffe6c3da2ee in 28ms, sequenceid=6, compaction requested=false 2024-11-25T02:54:53,357 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732503292666' 2024-11-25T02:54:53,358 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1b5a6ce08ca401de35fefffe6c3da2ee: 2024-11-25T02:54:53,459 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T02:54:54,297 DEBUG [hfile-prefetch-1732503290615 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8f5c75482ae948cc9276033c3fe9de04_0. 2024-11-25T02:54:54,298 DEBUG [hfile-prefetch-1732503290615 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8f5c75482ae948cc9276033c3fe9de04_117. 2024-11-25T02:54:54,298 DEBUG [hfile-prefetch-1732503290615 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/family/8f5c75482ae948cc9276033c3fe9de04, with total blocks 2, and data blocks 1 2024-11-25T02:54:54,298 DEBUG [hfile-prefetch-1732503290615 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-25T02:54:54,298 DEBUG [hfile-prefetch-1732503290615 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/family/8f5c75482ae948cc9276033c3fe9de04, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-25T02:54:54,299 DEBUG [hfile-prefetch-1732503290615 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/family/8f5c75482ae948cc9276033c3fe9de04. Will sleep 100ms and try the verification again. 2024-11-25T02:54:54,359 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;2c5ea3e41165:44817,5,FailOnTimeoutGroup] 2024-11-25T02:54:54,359 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2c5ea3e41165,44817,1732503291418' ***** 2024-11-25T02:54:54,359 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-25T02:54:54,359 INFO [RS:1;2c5ea3e41165:44817 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T02:54:54,370 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T02:54:54,373 INFO [regionserver/2c5ea3e41165:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T02:54:54,373 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T02:54:54,373 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T02:54:54,373 INFO [RS:1;2c5ea3e41165:44817 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/bucket.persistence 2024-11-25T02:54:54,373 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T02:54:54,373 INFO [RS:1;2c5ea3e41165:44817 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T02:54:54,373 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T02:54:54,383 DEBUG [RS:1;2c5ea3e41165:44817 {}] bucket.BucketCache(1487): Thread RS:1;2c5ea3e41165:44817 started persisting bucket cache to file 2024-11-25T02:54:54,383 DEBUG [RS:1;2c5ea3e41165:44817 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T02:54:54,384 DEBUG [RS:1;2c5ea3e41165:44817 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T02:54:54,387 DEBUG [RS:1;2c5ea3e41165:44817 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-25T02:54:54,387 DEBUG [RS:1;2c5ea3e41165:44817 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732503294383 2024-11-25T02:54:54,388 DEBUG [RS:1;2c5ea3e41165:44817 {}] bucket.BucketCache(1509): Thread RS:1;2c5ea3e41165:44817 finished persisting bucket cache to file, renaming 2024-11-25T02:54:54,388 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T02:54:54,388 INFO [RS:1;2c5ea3e41165:44817 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T02:54:54,388 INFO [RS:1;2c5ea3e41165:44817 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T02:54:54,388 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.HRegionServer(3091): Received CLOSE for 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:54,388 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.HRegionServer(959): stopping server 2c5ea3e41165,44817,1732503291418 2024-11-25T02:54:54,388 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T02:54:54,389 INFO [RS:1;2c5ea3e41165:44817 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2c5ea3e41165:44817. 2024-11-25T02:54:54,389 INFO [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T02:54:54,389 DEBUG [RS:1;2c5ea3e41165:44817 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T02:54:54,389 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1b5a6ce08ca401de35fefffe6c3da2ee, disabling compactions & flushes 2024-11-25T02:54:54,389 DEBUG [RS:1;2c5ea3e41165:44817 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:54,389 INFO [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:54,389 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:54,389 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T02:54:54,389 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. after waiting 0 ms 2024-11-25T02:54:54,389 DEBUG [RS:1;2c5ea3e41165:44817 {}] regionserver.HRegionServer(1325): Online Regions={1b5a6ce08ca401de35fefffe6c3da2ee=testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee.} 2024-11-25T02:54:54,389 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:54,389 DEBUG [RS:1;2c5ea3e41165:44817 {}] regionserver.HRegionServer(1351): Waiting on 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:54,390 DEBUG [StoreFileCloser-1b5a6ce08ca401de35fefffe6c3da2ee-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/family/8f5c75482ae948cc9276033c3fe9de04 2024-11-25T02:54:54,393 WARN [hfile-prefetch-1732503290615 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/family/8f5c75482ae948cc9276033c3fe9de04, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-25T02:54:54,395 DEBUG [hfile-prefetch-1732503290615 {}] hfile.PrefetchExecutor(121): Prefetch completed for 8f5c75482ae948cc9276033c3fe9de04 2024-11-25T02:54:54,395 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T02:54:54,395 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T02:54:54,395 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T02:54:54,395 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T02:54:54,397 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-25T02:54:54,397 INFO [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:54,397 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1b5a6ce08ca401de35fefffe6c3da2ee: Waiting for close lock at 1732503294389Running coprocessor pre-close hooks at 1732503294389Disabling compacts and flushes for region at 1732503294389Disabling writes for close at 1732503294389Writing region close event to WAL at 1732503294391 (+2 ms)Running coprocessor post-close hooks at 1732503294397 (+6 ms)Closed at 1732503294397 2024-11-25T02:54:54,397 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:54,399 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T02:54:54,399 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732503294395 2024-11-25T02:54:54,399 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T02:54:54,590 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.HRegionServer(976): stopping server 2c5ea3e41165,44817,1732503291418; all regions closed. 2024-11-25T02:54:54,592 DEBUG [RS:1;2c5ea3e41165:44817 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/oldWALs 2024-11-25T02:54:54,592 INFO [RS:1;2c5ea3e41165:44817 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2c5ea3e41165%2C44817%2C1732503291418:(num 1732503292118) 2024-11-25T02:54:54,592 DEBUG [RS:1;2c5ea3e41165:44817 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:54,592 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T02:54:54,592 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T02:54:54,592 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.ChoreService(370): Chore service for: regionserver/2c5ea3e41165:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T02:54:54,592 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T02:54:54,592 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T02:54:54,592 INFO [regionserver/2c5ea3e41165:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T02:54:54,592 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T02:54:54,592 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T02:54:54,593 INFO [RS:1;2c5ea3e41165:44817 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44817 2024-11-25T02:54:54,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2c5ea3e41165,44817,1732503291418 2024-11-25T02:54:54,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T02:54:54,628 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T02:54:54,640 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2c5ea3e41165,44817,1732503291418] 2024-11-25T02:54:54,647 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2c5ea3e41165,44817,1732503291418 already deleted, retry=false 2024-11-25T02:54:54,648 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 2c5ea3e41165,44817,1732503291418 on 2c5ea3e41165,43523,1732503291226 2024-11-25T02:54:54,655 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 2c5ea3e41165,44817,1732503291418, splitWal=true, meta=false 2024-11-25T02:54:54,658 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 2c5ea3e41165,44817,1732503291418 (carryingMeta=false) 2c5ea3e41165,44817,1732503291418/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4058c364[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-25T02:54:54,659 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 2c5ea3e41165,44817,1732503291418, splitWal=true, meta=false 2024-11-25T02:54:54,660 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 2c5ea3e41165,44817,1732503291418 had 1 regions 2024-11-25T02:54:54,661 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 2c5ea3e41165,44817,1732503291418, splitWal=true, meta=false, isMeta: false 2024-11-25T02:54:54,662 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,44817,1732503291418-splitting 2024-11-25T02:54:54,662 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,44817,1732503291418-splitting dir is empty, no logs to split. 2024-11-25T02:54:54,662 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2c5ea3e41165,44817,1732503291418 WAL count=0, meta=false 2024-11-25T02:54:54,664 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,44817,1732503291418-splitting dir is empty, no logs to split. 2024-11-25T02:54:54,665 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2c5ea3e41165,44817,1732503291418 WAL count=0, meta=false 2024-11-25T02:54:54,665 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 2c5ea3e41165,44817,1732503291418 WAL splitting is done? wals=0, meta=false 2024-11-25T02:54:54,665 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 2c5ea3e41165,44817,1732503291418 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,44817,1732503291418-splitting does not exist 2024-11-25T02:54:54,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732503292666, region=1b5a6ce08ca401de35fefffe6c3da2ee, ASSIGN}] 2024-11-25T02:54:54,669 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732503292666, region=1b5a6ce08ca401de35fefffe6c3da2ee, ASSIGN 2024-11-25T02:54:54,669 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732503292666, region=1b5a6ce08ca401de35fefffe6c3da2ee, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T02:54:54,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T02:54:54,739 INFO [RS:1;2c5ea3e41165:44817 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T02:54:54,740 INFO [RS:1;2c5ea3e41165:44817 {}] regionserver.HRegionServer(1031): Exiting; stopping=2c5ea3e41165,44817,1732503291418; zookeeper connection closed. 2024-11-25T02:54:54,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44817-0x1017133d8110002, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T02:54:54,740 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6c30c1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6c30c1 2024-11-25T02:54:54,820 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1b5a6ce08ca401de35fefffe6c3da2ee, regionState=OPENING, regionLocation=2c5ea3e41165,34583,1732503291374 2024-11-25T02:54:54,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732503292666, region=1b5a6ce08ca401de35fefffe6c3da2ee, ASSIGN because future has completed 2024-11-25T02:54:54,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b5a6ce08ca401de35fefffe6c3da2ee, server=2c5ea3e41165,34583,1732503291374}] 2024-11-25T02:54:54,878 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2c5ea3e41165:0 server-side Connection retries=45 2024-11-25T02:54:54,878 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:54,878 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:54,878 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T02:54:54,878 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T02:54:54,878 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T02:54:54,879 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T02:54:54,879 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T02:54:54,879 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43261 2024-11-25T02:54:54,881 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43261 connecting to ZooKeeper ensemble=127.0.0.1:51409 2024-11-25T02:54:54,881 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:54,882 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:54,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:432610x0, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T02:54:54,906 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43261-0x1017133d8110003 connected 2024-11-25T02:54:54,906 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:43261-0x1017133d8110003, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-25T02:54:54,906 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-25T02:54:54,907 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T02:54:54,908 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/bucket.cache 2024-11-25T02:54:54,908 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T02:54:54,908 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T02:54:54,908 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T02:54:54,908 INFO [Thread-201 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T02:54:54,909 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T02:54:54,909 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T02:54:54,910 INFO [Thread-201 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-25T02:54:54,910 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:43261-0x1017133d8110003, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T02:54:54,910 INFO [Thread-201 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-25T02:54:54,911 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43261-0x1017133d8110003, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T02:54:54,912 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43261 2024-11-25T02:54:54,912 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43261 2024-11-25T02:54:54,912 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43261 2024-11-25T02:54:54,913 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43261 2024-11-25T02:54:54,913 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43261 2024-11-25T02:54:54,913 DEBUG [Thread-201 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-25T02:54:54,914 INFO [Thread-201 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T02:54:54,914 INFO [Thread-201 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-25T02:54:54,914 INFO [Thread-201 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/bucket.persistence is complete. 2024-11-25T02:54:54,914 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.HRegionServer(746): ClusterId : 45c415a0-0ab9-4b81-9f1c-83473716c96b 2024-11-25T02:54:54,914 DEBUG [RS:2;2c5ea3e41165:43261 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T02:54:54,923 DEBUG [RS:2;2c5ea3e41165:43261 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T02:54:54,923 DEBUG [RS:2;2c5ea3e41165:43261 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T02:54:54,932 DEBUG [RS:2;2c5ea3e41165:43261 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T02:54:54,932 DEBUG [RS:2;2c5ea3e41165:43261 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67512097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2c5ea3e41165/172.17.0.2:0 2024-11-25T02:54:54,946 DEBUG [RS:2;2c5ea3e41165:43261 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;2c5ea3e41165:43261 2024-11-25T02:54:54,946 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T02:54:54,946 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T02:54:54,946 DEBUG [RS:2;2c5ea3e41165:43261 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T02:54:54,947 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.HRegionServer(2659): reportForDuty to master=2c5ea3e41165,43523,1732503291226 with port=43261, startcode=1732503294878 2024-11-25T02:54:54,947 DEBUG [RS:2;2c5ea3e41165:43261 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T02:54:54,949 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36261, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T02:54:54,950 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43523 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2c5ea3e41165,43261,1732503294878 2024-11-25T02:54:54,950 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43523 {}] master.ServerManager(517): Registering regionserver=2c5ea3e41165,43261,1732503294878 2024-11-25T02:54:54,951 DEBUG [RS:2;2c5ea3e41165:43261 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a 2024-11-25T02:54:54,952 DEBUG [RS:2;2c5ea3e41165:43261 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T02:54:54,952 DEBUG [RS:2;2c5ea3e41165:43261 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T02:54:54,978 INFO [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:54,978 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b5a6ce08ca401de35fefffe6c3da2ee, NAME => 'testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee.', STARTKEY => '', ENDKEY => ''} 2024-11-25T02:54:54,979 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732503292666 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:54,979 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T02:54:54,979 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:54,979 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:54,980 INFO [StoreOpener-1b5a6ce08ca401de35fefffe6c3da2ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:54,980 INFO [StoreOpener-1b5a6ce08ca401de35fefffe6c3da2ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b5a6ce08ca401de35fefffe6c3da2ee columnFamilyName family 2024-11-25T02:54:54,980 DEBUG [StoreOpener-1b5a6ce08ca401de35fefffe6c3da2ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T02:54:54,982 DEBUG [StoreFileOpener-1b5a6ce08ca401de35fefffe6c3da2ee-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/family/8f5c75482ae948cc9276033c3fe9de04, delay=915 ms 2024-11-25T02:54:54,983 DEBUG [StoreOpener-1b5a6ce08ca401de35fefffe6c3da2ee-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/family/8f5c75482ae948cc9276033c3fe9de04 2024-11-25T02:54:54,984 INFO [StoreOpener-1b5a6ce08ca401de35fefffe6c3da2ee-1 {}] regionserver.HStore(327): Store=1b5a6ce08ca401de35fefffe6c3da2ee/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T02:54:54,984 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:54,985 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:54,985 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:54,985 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:54,985 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:54,986 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:54,987 INFO [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 1b5a6ce08ca401de35fefffe6c3da2ee; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70855237, jitterRate=0.055825307965278625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T02:54:54,987 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:54,988 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 1b5a6ce08ca401de35fefffe6c3da2ee: Running coprocessor pre-open hook at 1732503294979Writing region info on filesystem at 1732503294979Initializing all the Stores at 1732503294979Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732503294979Cleaning up temporary data from old regions at 1732503294985 (+6 ms)Running coprocessor post-open hooks at 1732503294987 (+2 ms)Region opened successfully at 1732503294988 (+1 ms) 2024-11-25T02:54:54,988 INFO [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee., pid=9, masterSystemTime=1732503294976 2024-11-25T02:54:54,991 DEBUG [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:54,991 INFO [RS_OPEN_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:54,991 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1b5a6ce08ca401de35fefffe6c3da2ee, regionState=OPEN, openSeqNum=10, regionLocation=2c5ea3e41165,34583,1732503291374 2024-11-25T02:54:54,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b5a6ce08ca401de35fefffe6c3da2ee, server=2c5ea3e41165,34583,1732503291374 because future has completed 2024-11-25T02:54:54,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-25T02:54:54,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1b5a6ce08ca401de35fefffe6c3da2ee, server=2c5ea3e41165,34583,1732503291374 in 173 msec 2024-11-25T02:54:55,000 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T02:54:55,000 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732503292666, region=1b5a6ce08ca401de35fefffe6c3da2ee, ASSIGN in 331 msec 2024-11-25T02:54:55,000 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server 2c5ea3e41165,44817,1732503291418 after splitting done 2024-11-25T02:54:55,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 2c5ea3e41165,44817,1732503291418, splitWal=true, meta=false in 351 msec 2024-11-25T02:54:55,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T02:54:55,046 DEBUG [RS:2;2c5ea3e41165:43261 {}] zookeeper.ZKUtil(111): regionserver:43261-0x1017133d8110003, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2c5ea3e41165,43261,1732503294878 2024-11-25T02:54:55,046 WARN [RS:2;2c5ea3e41165:43261 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T02:54:55,046 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2c5ea3e41165,43261,1732503294878] 2024-11-25T02:54:55,046 INFO [RS:2;2c5ea3e41165:43261 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T02:54:55,046 DEBUG [RS:2;2c5ea3e41165:43261 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,43261,1732503294878 2024-11-25T02:54:55,051 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T02:54:55,054 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T02:54:55,055 INFO [RS:2;2c5ea3e41165:43261 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T02:54:55,055 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:55,055 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T02:54:55,056 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T02:54:55,056 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:55,056 DEBUG [RS:2;2c5ea3e41165:43261 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:55,056 DEBUG [RS:2;2c5ea3e41165:43261 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:55,056 DEBUG [RS:2;2c5ea3e41165:43261 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:55,056 DEBUG [RS:2;2c5ea3e41165:43261 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:55,056 DEBUG [RS:2;2c5ea3e41165:43261 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:55,057 DEBUG [RS:2;2c5ea3e41165:43261 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2c5ea3e41165:0, corePoolSize=2, maxPoolSize=2 2024-11-25T02:54:55,057 DEBUG [RS:2;2c5ea3e41165:43261 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:55,057 DEBUG [RS:2;2c5ea3e41165:43261 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:55,057 DEBUG [RS:2;2c5ea3e41165:43261 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:55,057 DEBUG [RS:2;2c5ea3e41165:43261 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:55,057 DEBUG [RS:2;2c5ea3e41165:43261 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:55,057 DEBUG [RS:2;2c5ea3e41165:43261 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2c5ea3e41165:0, corePoolSize=1, maxPoolSize=1 2024-11-25T02:54:55,057 DEBUG [RS:2;2c5ea3e41165:43261 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2c5ea3e41165:0, corePoolSize=3, maxPoolSize=3 2024-11-25T02:54:55,057 DEBUG [RS:2;2c5ea3e41165:43261 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2c5ea3e41165:0, corePoolSize=3, maxPoolSize=3 2024-11-25T02:54:55,058 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:55,058 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:55,058 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:55,058 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:55,058 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:55,058 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,43261,1732503294878-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T02:54:55,073 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T02:54:55,074 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.ChoreService(168): Chore ScheduledChore name=2c5ea3e41165,43261,1732503294878-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:55,074 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:55,074 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.Replication(171): 2c5ea3e41165,43261,1732503294878 started 2024-11-25T02:54:55,088 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T02:54:55,088 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.HRegionServer(1482): Serving as 2c5ea3e41165,43261,1732503294878, RpcServer on 2c5ea3e41165/172.17.0.2:43261, sessionid=0x1017133d8110003 2024-11-25T02:54:55,088 DEBUG [RS:2;2c5ea3e41165:43261 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T02:54:55,089 DEBUG [RS:2;2c5ea3e41165:43261 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2c5ea3e41165,43261,1732503294878 2024-11-25T02:54:55,089 DEBUG [RS:2;2c5ea3e41165:43261 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2c5ea3e41165,43261,1732503294878' 2024-11-25T02:54:55,089 DEBUG [RS:2;2c5ea3e41165:43261 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T02:54:55,089 DEBUG [RS:2;2c5ea3e41165:43261 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T02:54:55,090 DEBUG [RS:2;2c5ea3e41165:43261 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T02:54:55,090 DEBUG [RS:2;2c5ea3e41165:43261 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T02:54:55,090 DEBUG [RS:2;2c5ea3e41165:43261 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2c5ea3e41165,43261,1732503294878 2024-11-25T02:54:55,090 DEBUG [RS:2;2c5ea3e41165:43261 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2c5ea3e41165,43261,1732503294878' 2024-11-25T02:54:55,090 DEBUG [RS:2;2c5ea3e41165:43261 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T02:54:55,091 DEBUG [RS:2;2c5ea3e41165:43261 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T02:54:55,091 DEBUG [RS:2;2c5ea3e41165:43261 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T02:54:55,091 INFO [RS:2;2c5ea3e41165:43261 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T02:54:55,091 INFO [RS:2;2c5ea3e41165:43261 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T02:54:55,192 INFO [RS:2;2c5ea3e41165:43261 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T02:54:55,193 INFO [RS:2;2c5ea3e41165:43261 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=2c5ea3e41165%2C43261%2C1732503294878, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,43261,1732503294878, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/oldWALs, maxLogs=55 2024-11-25T02:54:55,210 INFO [RS:2;2c5ea3e41165:43261 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/WALs/2c5ea3e41165,43261,1732503294878/2c5ea3e41165%2C43261%2C1732503294878.1732503295193 2024-11-25T02:54:55,210 DEBUG [RS:2;2c5ea3e41165:43261 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T02:54:55,387 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T02:54:55,388 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T02:54:55,390 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732503292666 2024-11-25T02:54:55,391 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732503292666 Metrics about Tables on a single HBase RegionServer 2024-11-25T02:54:55,392 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T02:54:55,392 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T02:54:55,393 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T02:54:55,393 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T02:54:55,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T02:54:55,589 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T02:54:55,589 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T02:54:55,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:55,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:55,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T02:54:55,589 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T02:54:55,589 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=376538664, stopped=false 2024-11-25T02:54:55,589 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2c5ea3e41165,43523,1732503291226 2024-11-25T02:54:55,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43261-0x1017133d8110003, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T02:54:55,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T02:54:55,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T02:54:55,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:55,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:55,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43261-0x1017133d8110003, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:55,637 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T02:54:55,637 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T02:54:55,637 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T02:54:55,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:55,638 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43261-0x1017133d8110003, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T02:54:55,638 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T02:54:55,638 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T02:54:55,638 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2c5ea3e41165,34583,1732503291374' ***** 2024-11-25T02:54:55,638 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T02:54:55,638 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2c5ea3e41165,43261,1732503294878' ***** 2024-11-25T02:54:55,638 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T02:54:55,638 INFO [RS:0;2c5ea3e41165:34583 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T02:54:55,639 INFO [RS:0;2c5ea3e41165:34583 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/bucket.persistence 2024-11-25T02:54:55,639 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T02:54:55,639 INFO [RS:2;2c5ea3e41165:43261 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T02:54:55,639 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T02:54:55,639 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T02:54:55,639 INFO [RS:2;2c5ea3e41165:43261 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/cbe629b7-97e3-91f3-019b-82ebe8721feb/bucket.persistence 2024-11-25T02:54:55,639 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T02:54:55,639 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T02:54:55,639 INFO [RS:0;2c5ea3e41165:34583 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T02:54:55,639 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T02:54:55,640 INFO [RS:2;2c5ea3e41165:43261 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T02:54:55,639 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T02:54:55,640 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T02:54:55,649 DEBUG [RS:0;2c5ea3e41165:34583 {}] bucket.BucketCache(1487): Thread RS:0;2c5ea3e41165:34583 started persisting bucket cache to file 2024-11-25T02:54:55,650 DEBUG [RS:0;2c5ea3e41165:34583 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T02:54:55,650 DEBUG [RS:0;2c5ea3e41165:34583 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T02:54:55,650 DEBUG [RS:2;2c5ea3e41165:43261 {}] bucket.BucketCache(1487): Thread RS:2;2c5ea3e41165:43261 started persisting bucket cache to file 2024-11-25T02:54:55,650 DEBUG [RS:2;2c5ea3e41165:43261 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T02:54:55,650 DEBUG [RS:2;2c5ea3e41165:43261 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T02:54:55,653 DEBUG [RS:2;2c5ea3e41165:43261 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T02:54:55,653 DEBUG [RS:0;2c5ea3e41165:34583 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T02:54:55,653 DEBUG [RS:2;2c5ea3e41165:43261 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732503295650 2024-11-25T02:54:55,653 DEBUG [RS:0;2c5ea3e41165:34583 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732503295649 2024-11-25T02:54:55,653 DEBUG [RS:2;2c5ea3e41165:43261 {}] bucket.BucketCache(1509): Thread RS:2;2c5ea3e41165:43261 finished persisting bucket cache to file, renaming 2024-11-25T02:54:55,653 DEBUG [RS:0;2c5ea3e41165:34583 {}] bucket.BucketCache(1509): Thread RS:0;2c5ea3e41165:34583 finished persisting bucket cache to file, renaming 2024-11-25T02:54:55,653 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T02:54:55,654 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T02:54:55,654 INFO [RS:0;2c5ea3e41165:34583 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T02:54:55,654 INFO [RS:0;2c5ea3e41165:34583 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T02:54:55,654 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T02:54:55,654 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer(3091): Received CLOSE for 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:55,654 INFO [RS:2;2c5ea3e41165:43261 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T02:54:55,654 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T02:54:55,654 INFO [RS:2;2c5ea3e41165:43261 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T02:54:55,654 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer(959): stopping server 2c5ea3e41165,34583,1732503291374 2024-11-25T02:54:55,654 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.HRegionServer(959): stopping server 2c5ea3e41165,43261,1732503294878 2024-11-25T02:54:55,654 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T02:54:55,654 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T02:54:55,654 INFO [RS:2;2c5ea3e41165:43261 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;2c5ea3e41165:43261. 2024-11-25T02:54:55,654 INFO [RS:0;2c5ea3e41165:34583 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2c5ea3e41165:34583. 2024-11-25T02:54:55,654 DEBUG [RS:2;2c5ea3e41165:43261 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T02:54:55,654 INFO [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T02:54:55,655 DEBUG [RS:0;2c5ea3e41165:34583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T02:54:55,655 DEBUG [RS:2;2c5ea3e41165:43261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:55,655 DEBUG [RS:0;2c5ea3e41165:34583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:55,655 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1b5a6ce08ca401de35fefffe6c3da2ee, disabling compactions & flushes 2024-11-25T02:54:55,655 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.HRegionServer(976): stopping server 2c5ea3e41165,43261,1732503294878; all regions closed. 2024-11-25T02:54:55,655 INFO [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:55,655 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T02:54:55,655 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T02:54:55,655 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:55,655 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T02:54:55,655 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. after waiting 0 ms 2024-11-25T02:54:55,655 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T02:54:55,655 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:55,655 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-25T02:54:55,656 DEBUG [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer(1325): Online Regions={1b5a6ce08ca401de35fefffe6c3da2ee=testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee., 1588230740=hbase:meta,,1.1588230740} 2024-11-25T02:54:55,656 DEBUG [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1b5a6ce08ca401de35fefffe6c3da2ee 2024-11-25T02:54:55,656 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T02:54:55,656 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T02:54:55,656 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T02:54:55,656 DEBUG [StoreFileCloser-1b5a6ce08ca401de35fefffe6c3da2ee-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/family/8f5c75482ae948cc9276033c3fe9de04 2024-11-25T02:54:55,656 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T02:54:55,656 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T02:54:55,656 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T02:54:55,657 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-25T02:54:55,658 DEBUG [RS:2;2c5ea3e41165:43261 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/oldWALs 2024-11-25T02:54:55,658 INFO [RS:2;2c5ea3e41165:43261 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2c5ea3e41165%2C43261%2C1732503294878:(num 1732503295193) 2024-11-25T02:54:55,658 DEBUG [RS:2;2c5ea3e41165:43261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:55,658 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T02:54:55,659 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T02:54:55,659 INFO [regionserver/2c5ea3e41165:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T02:54:55,659 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.ChoreService(370): Chore service for: regionserver/2c5ea3e41165:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T02:54:55,659 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T02:54:55,659 INFO [regionserver/2c5ea3e41165:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T02:54:55,659 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T02:54:55,660 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T02:54:55,660 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T02:54:55,660 INFO [RS:2;2c5ea3e41165:43261 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43261 2024-11-25T02:54:55,663 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/default/testBlockEvictionOnGracefulStop1732503292666/1b5a6ce08ca401de35fefffe6c3da2ee/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-25T02:54:55,663 INFO [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:55,663 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1b5a6ce08ca401de35fefffe6c3da2ee: Waiting for close lock at 1732503295654Running coprocessor pre-close hooks at 1732503295655 (+1 ms)Disabling compacts and flushes for region at 1732503295655Disabling writes for close at 1732503295655Writing region close event to WAL at 1732503295656 (+1 ms)Running coprocessor post-close hooks at 1732503295663 (+7 ms)Closed at 1732503295663 2024-11-25T02:54:55,663 DEBUG [RS_CLOSE_REGION-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee. 2024-11-25T02:54:55,667 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5d91f6f22ca44ad1a41583aa399ca47e 2024-11-25T02:54:55,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T02:54:55,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43261-0x1017133d8110003, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2c5ea3e41165,43261,1732503294878 2024-11-25T02:54:55,675 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T02:54:55,676 INFO [regionserver/2c5ea3e41165:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T02:54:55,681 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/info/5d91f6f22ca44ad1a41583aa399ca47e is 209, key is testBlockEvictionOnGracefulStop1732503292666,,1732503292671.1b5a6ce08ca401de35fefffe6c3da2ee./info:regioninfo/1732503294991/Put/seqid=0 2024-11-25T02:54:55,682 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/info/5d91f6f22ca44ad1a41583aa399ca47e 2024-11-25T02:54:55,689 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2c5ea3e41165,43261,1732503294878] 2024-11-25T02:54:55,690 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 21853b41ce4f4b428ab107e8fd4ce9db 2024-11-25T02:54:55,697 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2c5ea3e41165,43261,1732503294878 already deleted, retry=false 2024-11-25T02:54:55,697 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2c5ea3e41165,43261,1732503294878 expired; onlineServers=1 2024-11-25T02:54:55,702 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/ns/21853b41ce4f4b428ab107e8fd4ce9db is 43, key is default/ns:d/1732503292560/Put/seqid=0 2024-11-25T02:54:55,702 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/ns/21853b41ce4f4b428ab107e8fd4ce9db 2024-11-25T02:54:55,709 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d53d7d1cf018426ebefd11526af2b9cc 2024-11-25T02:54:55,721 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/table/d53d7d1cf018426ebefd11526af2b9cc is 80, key is testBlockEvictionOnGracefulStop1732503292666/table:state/1732503293044/Put/seqid=0 2024-11-25T02:54:55,721 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/table/d53d7d1cf018426ebefd11526af2b9cc 2024-11-25T02:54:55,723 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/info/5d91f6f22ca44ad1a41583aa399ca47e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/info/5d91f6f22ca44ad1a41583aa399ca47e 2024-11-25T02:54:55,724 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/info/5d91f6f22ca44ad1a41583aa399ca47e, delay=917 ms 2024-11-25T02:54:55,725 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/info/5d91f6f22ca44ad1a41583aa399ca47e, entries=14, sequenceid=13, filesize=7.5 K 2024-11-25T02:54:55,726 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/ns/21853b41ce4f4b428ab107e8fd4ce9db as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/ns/21853b41ce4f4b428ab107e8fd4ce9db 2024-11-25T02:54:55,727 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/ns/21853b41ce4f4b428ab107e8fd4ce9db, delay=995 ms 2024-11-25T02:54:55,728 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/ns/21853b41ce4f4b428ab107e8fd4ce9db, entries=2, sequenceid=13, filesize=5.0 K 2024-11-25T02:54:55,728 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/.tmp/table/d53d7d1cf018426ebefd11526af2b9cc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/table/d53d7d1cf018426ebefd11526af2b9cc 2024-11-25T02:54:55,729 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/table/d53d7d1cf018426ebefd11526af2b9cc, delay=962 ms 2024-11-25T02:54:55,730 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/table/d53d7d1cf018426ebefd11526af2b9cc, entries=2, sequenceid=13, filesize=5.3 K 2024-11-25T02:54:55,730 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=13, compaction requested=false 2024-11-25T02:54:55,731 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/info/5d91f6f22ca44ad1a41583aa399ca47e 2024-11-25T02:54:55,731 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/ns/21853b41ce4f4b428ab107e8fd4ce9db 2024-11-25T02:54:55,732 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/table/d53d7d1cf018426ebefd11526af2b9cc 2024-11-25T02:54:55,736 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-25T02:54:55,736 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T02:54:55,736 INFO [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T02:54:55,736 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732503295656Running coprocessor pre-close hooks at 1732503295656Disabling compacts and flushes for region at 1732503295656Disabling writes for close at 1732503295656Obtaining lock to block concurrent updates at 1732503295657 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732503295657Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732503295658 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732503295658Flushing 1588230740/info: creating writer at 1732503295658Flushing 1588230740/info: appending metadata at 1732503295681 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732503295681Flushing 1588230740/ns: creating writer at 1732503295685 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732503295701 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732503295701Flushing 1588230740/table: creating writer at 1732503295704 (+3 ms)Flushing 1588230740/table: appending metadata at 1732503295720 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732503295720Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@609ba027: reopening flushed file at 1732503295723 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d1c45f6: reopening flushed file at 1732503295725 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@712159e2: reopening flushed file at 1732503295728 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=13, compaction requested=false at 1732503295730 (+2 ms)Writing region close event to WAL at 1732503295732 (+2 ms)Running coprocessor post-close hooks at 1732503295736 (+4 ms)Closed at 1732503295736 2024-11-25T02:54:55,736 DEBUG [RS_CLOSE_META-regionserver/2c5ea3e41165:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T02:54:55,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43261-0x1017133d8110003, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T02:54:55,789 INFO [RS:2;2c5ea3e41165:43261 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T02:54:55,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43261-0x1017133d8110003, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T02:54:55,789 INFO [RS:2;2c5ea3e41165:43261 {}] regionserver.HRegionServer(1031): Exiting; stopping=2c5ea3e41165,43261,1732503294878; zookeeper connection closed. 2024-11-25T02:54:55,790 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@615971e3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@615971e3 2024-11-25T02:54:55,856 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer(976): stopping server 2c5ea3e41165,34583,1732503291374; all regions closed. 2024-11-25T02:54:55,859 DEBUG [RS:0;2c5ea3e41165:34583 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/oldWALs 2024-11-25T02:54:55,859 INFO [RS:0;2c5ea3e41165:34583 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2c5ea3e41165%2C34583%2C1732503291374.meta:.meta(num 1732503292482) 2024-11-25T02:54:55,861 DEBUG [RS:0;2c5ea3e41165:34583 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/oldWALs 2024-11-25T02:54:55,861 INFO [RS:0;2c5ea3e41165:34583 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2c5ea3e41165%2C34583%2C1732503291374:(num 1732503292110) 2024-11-25T02:54:55,862 DEBUG [RS:0;2c5ea3e41165:34583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T02:54:55,862 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T02:54:55,862 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T02:54:55,862 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.ChoreService(370): Chore service for: regionserver/2c5ea3e41165:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T02:54:55,863 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T02:54:55,863 INFO [regionserver/2c5ea3e41165:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T02:54:55,863 INFO [RS:0;2c5ea3e41165:34583 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34583 2024-11-25T02:54:55,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2c5ea3e41165,34583,1732503291374 2024-11-25T02:54:55,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T02:54:55,895 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T02:54:55,905 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2c5ea3e41165,34583,1732503291374] 2024-11-25T02:54:55,913 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2c5ea3e41165,34583,1732503291374 already deleted, retry=false 2024-11-25T02:54:55,913 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2c5ea3e41165,34583,1732503291374 expired; onlineServers=0 2024-11-25T02:54:55,914 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2c5ea3e41165,43523,1732503291226' ***** 2024-11-25T02:54:55,914 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T02:54:55,914 INFO [M:0;2c5ea3e41165:43523 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T02:54:55,914 INFO [M:0;2c5ea3e41165:43523 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T02:54:55,914 DEBUG [M:0;2c5ea3e41165:43523 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T02:54:55,914 DEBUG [M:0;2c5ea3e41165:43523 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T02:54:55,914 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T02:54:55,914 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster-HFileCleaner.large.0-1732503291831 {}] cleaner.HFileCleaner(306): Exit Thread[master/2c5ea3e41165:0:becomeActiveMaster-HFileCleaner.large.0-1732503291831,5,FailOnTimeoutGroup] 2024-11-25T02:54:55,914 DEBUG [master/2c5ea3e41165:0:becomeActiveMaster-HFileCleaner.small.0-1732503291831 {}] cleaner.HFileCleaner(306): Exit Thread[master/2c5ea3e41165:0:becomeActiveMaster-HFileCleaner.small.0-1732503291831,5,FailOnTimeoutGroup] 2024-11-25T02:54:55,914 INFO [M:0;2c5ea3e41165:43523 {}] hbase.ChoreService(370): Chore service for: master/2c5ea3e41165:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T02:54:55,915 INFO [M:0;2c5ea3e41165:43523 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T02:54:55,915 DEBUG [M:0;2c5ea3e41165:43523 {}] master.HMaster(1795): Stopping service threads 2024-11-25T02:54:55,915 INFO [M:0;2c5ea3e41165:43523 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T02:54:55,915 INFO [M:0;2c5ea3e41165:43523 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T02:54:55,915 INFO [M:0;2c5ea3e41165:43523 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T02:54:55,915 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T02:54:55,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T02:54:55,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T02:54:55,922 DEBUG [M:0;2c5ea3e41165:43523 {}] zookeeper.ZKUtil(347): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T02:54:55,922 WARN [M:0;2c5ea3e41165:43523 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T02:54:55,923 INFO [M:0;2c5ea3e41165:43523 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/.lastflushedseqids 2024-11-25T02:54:55,930 INFO [M:0;2c5ea3e41165:43523 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T02:54:55,930 INFO [M:0;2c5ea3e41165:43523 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T02:54:55,930 DEBUG [M:0;2c5ea3e41165:43523 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T02:54:55,930 INFO [M:0;2c5ea3e41165:43523 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T02:54:55,930 DEBUG [M:0;2c5ea3e41165:43523 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T02:54:55,930 DEBUG [M:0;2c5ea3e41165:43523 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T02:54:55,930 DEBUG [M:0;2c5ea3e41165:43523 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T02:54:55,930 INFO [M:0;2c5ea3e41165:43523 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.81 KB heapSize=41.05 KB 2024-11-25T02:54:55,939 DEBUG [M:0;2c5ea3e41165:43523 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1eb3da2060ba4ac5b651da521a7a9363 2024-11-25T02:54:55,952 DEBUG [M:0;2c5ea3e41165:43523 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1eb3da2060ba4ac5b651da521a7a9363 is 82, key is hbase:meta,,1/info:regioninfo/1732503292513/Put/seqid=0 2024-11-25T02:54:55,953 INFO [M:0;2c5ea3e41165:43523 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1eb3da2060ba4ac5b651da521a7a9363 2024-11-25T02:54:55,959 DEBUG [M:0;2c5ea3e41165:43523 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 94f5418b71b64cecb405e328924f0f29 2024-11-25T02:54:55,972 DEBUG [M:0;2c5ea3e41165:43523 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/94f5418b71b64cecb405e328924f0f29 is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732503293049/Put/seqid=0 2024-11-25T02:54:55,972 INFO [M:0;2c5ea3e41165:43523 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.08 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/94f5418b71b64cecb405e328924f0f29 2024-11-25T02:54:55,979 DEBUG [M:0;2c5ea3e41165:43523 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0d42606026d041cbaaa76744e4f3aafd 2024-11-25T02:54:55,992 DEBUG [M:0;2c5ea3e41165:43523 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0d42606026d041cbaaa76744e4f3aafd is 69, key is 2c5ea3e41165,34583,1732503291374/rs:state/1732503291908/Put/seqid=0 2024-11-25T02:54:55,992 INFO [M:0;2c5ea3e41165:43523 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0d42606026d041cbaaa76744e4f3aafd 2024-11-25T02:54:55,994 INFO [M:0;2c5ea3e41165:43523 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0d42606026d041cbaaa76744e4f3aafd 2024-11-25T02:54:55,994 DEBUG [M:0;2c5ea3e41165:43523 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1eb3da2060ba4ac5b651da521a7a9363 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1eb3da2060ba4ac5b651da521a7a9363 2024-11-25T02:54:55,996 INFO [M:0;2c5ea3e41165:43523 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1eb3da2060ba4ac5b651da521a7a9363, entries=8, sequenceid=83, filesize=5.5 K 2024-11-25T02:54:55,997 DEBUG [M:0;2c5ea3e41165:43523 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/94f5418b71b64cecb405e328924f0f29 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/94f5418b71b64cecb405e328924f0f29 2024-11-25T02:54:55,999 INFO [M:0;2c5ea3e41165:43523 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/94f5418b71b64cecb405e328924f0f29, entries=9, sequenceid=83, filesize=6.4 K 2024-11-25T02:54:55,999 DEBUG [M:0;2c5ea3e41165:43523 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0d42606026d041cbaaa76744e4f3aafd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0d42606026d041cbaaa76744e4f3aafd 2024-11-25T02:54:56,001 INFO [M:0;2c5ea3e41165:43523 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0d42606026d041cbaaa76744e4f3aafd 2024-11-25T02:54:56,001 INFO [M:0;2c5ea3e41165:43523 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/871e56b7-5518-3977-3cdb-5e4f7fab339f/b1757220-5976-bcc1-d0b7-63964b43bf6a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0d42606026d041cbaaa76744e4f3aafd, entries=3, sequenceid=83, filesize=5.3 K 2024-11-25T02:54:56,001 INFO [M:0;2c5ea3e41165:43523 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=83, compaction requested=false 2024-11-25T02:54:56,003 INFO [M:0;2c5ea3e41165:43523 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T02:54:56,003 DEBUG [M:0;2c5ea3e41165:43523 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732503295930Disabling compacts and flushes for region at 1732503295930Disabling writes for close at 1732503295930Obtaining lock to block concurrent updates at 1732503295930Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732503295930Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33602, getHeapSize=41976, getOffHeapSize=0, getCellsCount=98 at 1732503295931 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732503295931Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732503295931Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732503295952 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732503295952Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732503295954 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732503295971 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732503295971Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732503295974 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732503295991 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732503295991Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65c3b140: reopening flushed file at 1732503295994 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d3f647b: reopening flushed file at 1732503295996 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@375a78f5: reopening flushed file at 1732503295999 (+3 ms)Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=83, compaction requested=false at 1732503296001 (+2 ms)Writing region close event to WAL at 1732503296003 (+2 ms)Closed at 1732503296003 2024-11-25T02:54:56,004 INFO [M:0;2c5ea3e41165:43523 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T02:54:56,004 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T02:54:56,004 INFO [M:0;2c5ea3e41165:43523 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43523 2024-11-25T02:54:56,004 INFO [M:0;2c5ea3e41165:43523 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T02:54:56,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T02:54:56,005 INFO [RS:0;2c5ea3e41165:34583 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T02:54:56,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34583-0x1017133d8110001, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T02:54:56,005 INFO [RS:0;2c5ea3e41165:34583 {}] regionserver.HRegionServer(1031): Exiting; stopping=2c5ea3e41165,34583,1732503291374; zookeeper connection closed. 2024-11-25T02:54:56,006 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3179dc01 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3179dc01 2024-11-25T02:54:56,006 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-25T02:54:56,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T02:54:56,131 INFO [M:0;2c5ea3e41165:43523 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T02:54:56,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43523-0x1017133d8110000, quorum=127.0.0.1:51409, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T02:54:56,137 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T02:54:56,139 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-25T02:54:56,139 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T02:54:56,146 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=73 (was 51) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=366 (was 398), ProcessCount=11 (was 11), AvailableMemoryMB=12000 (was 12132)