2024-11-28 01:44:59,958 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-28 01:44:59,979 main DEBUG Took 0.018987 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-28 01:44:59,979 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-28 01:44:59,980 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-28 01:44:59,981 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-28 01:44:59,982 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:44:59,989 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-28 01:45:00,002 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:45:00,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:45:00,004 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:45:00,004 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:45:00,005 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:45:00,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:45:00,006 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:45:00,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:45:00,007 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:45:00,007 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:45:00,008 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:45:00,008 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:45:00,009 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:45:00,009 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:45:00,009 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:45:00,009 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:45:00,010 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:45:00,010 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:45:00,011 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:45:00,011 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:45:00,011 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:45:00,012 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:45:00,012 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:45:00,012 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 01:45:00,013 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:45:00,013 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-28 01:45:00,014 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 01:45:00,016 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-28 01:45:00,017 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-28 01:45:00,018 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-28 01:45:00,019 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-28 01:45:00,019 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-28 01:45:00,031 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-28 01:45:00,035 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-28 01:45:00,037 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-28 01:45:00,037 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-28 01:45:00,038 main DEBUG createAppenders(={Console}) 2024-11-28 01:45:00,038 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-28 01:45:00,039 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-28 01:45:00,039 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-28 01:45:00,040 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-28 01:45:00,041 main DEBUG OutputStream closed 2024-11-28 01:45:00,041 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-28 01:45:00,041 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-28 01:45:00,042 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-28 01:45:00,131 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-28 01:45:00,134 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-28 01:45:00,135 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-28 01:45:00,136 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-28 01:45:00,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-28 01:45:00,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-28 01:45:00,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-28 01:45:00,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-28 01:45:00,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-28 01:45:00,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-28 01:45:00,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-28 01:45:00,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-28 01:45:00,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-28 01:45:00,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-28 01:45:00,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-28 01:45:00,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-28 01:45:00,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-28 01:45:00,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-28 01:45:00,146 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28 01:45:00,146 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-28 01:45:00,146 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-28 01:45:00,147 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-28T01:45:00,162 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-28 01:45:00,164 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-28 01:45:00,165 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28T01:45:00,412 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae 2024-11-28T01:45:00,437 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=289, ProcessCount=11, AvailableMemoryMB=7918 2024-11-28T01:45:00,553 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-28T01:45:00,646 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:00,663 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/cluster_8a2f2c5e-9318-2239-36bf-b022a1997a79, deleteOnExit=true 2024-11-28T01:45:00,769 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/cluster_8a2f2c5e-9318-2239-36bf-b022a1997a79/zookeeper_0, clientPort=54192, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/cluster_8a2f2c5e-9318-2239-36bf-b022a1997a79/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/cluster_8a2f2c5e-9318-2239-36bf-b022a1997a79/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-28T01:45:00,781 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54192 2024-11-28T01:45:00,782 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:00,870 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c with version=8 2024-11-28T01:45:00,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/hbase-staging 2024-11-28T01:45:00,970 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-28T01:45:01,263 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9df3f25f5867:0 server-side Connection retries=45 2024-11-28T01:45:01,274 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:01,276 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:01,287 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T01:45:01,288 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:01,288 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T01:45:01,452 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-28T01:45:01,542 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T01:45:01,581 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-28T01:45:01,593 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-28T01:45:01,597 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T01:45:01,643 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 12967 (auto-detected) 2024-11-28T01:45:01,645 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-28T01:45:01,673 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40245 2024-11-28T01:45:01,708 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40245 connecting to ZooKeeper ensemble=127.0.0.1:54192 2024-11-28T01:45:01,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:402450x0, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T01:45:01,753 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40245-0x1003bcd212c0000 connected 2024-11-28T01:45:01,780 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:01,781 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:01,789 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T01:45:01,793 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c, hbase.cluster.distributed=false 2024-11-28T01:45:01,819 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T01:45:01,828 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40245 2024-11-28T01:45:01,828 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40245 2024-11-28T01:45:01,829 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40245 2024-11-28T01:45:01,831 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40245 2024-11-28T01:45:01,832 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40245 2024-11-28T01:45:01,946 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9df3f25f5867:0 server-side Connection retries=45 2024-11-28T01:45:01,948 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:01,948 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:01,949 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T01:45:01,949 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:01,949 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T01:45:01,952 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T01:45:01,955 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T01:45:01,956 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35035 2024-11-28T01:45:01,958 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35035 connecting to ZooKeeper ensemble=127.0.0.1:54192 2024-11-28T01:45:01,959 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:01,960 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:01,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:350350x0, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T01:45:01,965 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35035-0x1003bcd212c0001 connected 2024-11-28T01:45:01,965 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T01:45:01,972 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T01:45:01,985 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/bucket.cache 2024-11-28T01:45:01,993 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-28T01:45:01,995 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-28T01:45:01,997 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-28T01:45:01,999 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-28T01:45:02,000 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-28T01:45:02,007 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-28T01:45:02,011 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-28T01:45:02,012 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/bucket.persistence is complete. 2024-11-28T01:45:02,014 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T01:45:02,016 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T01:45:02,023 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T01:45:02,026 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35035 2024-11-28T01:45:02,027 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35035 2024-11-28T01:45:02,027 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35035 2024-11-28T01:45:02,029 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35035 2024-11-28T01:45:02,030 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35035 2024-11-28T01:45:02,060 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9df3f25f5867:0 server-side Connection retries=45 2024-11-28T01:45:02,060 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:02,060 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:02,061 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T01:45:02,061 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:02,061 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T01:45:02,061 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T01:45:02,062 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T01:45:02,063 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32859 2024-11-28T01:45:02,066 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32859 connecting to ZooKeeper ensemble=127.0.0.1:54192 2024-11-28T01:45:02,067 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:02,068 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:02,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:328590x0, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T01:45:02,073 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:328590x0, quorum=127.0.0.1:54192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T01:45:02,073 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32859-0x1003bcd212c0002 connected 2024-11-28T01:45:02,074 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T01:45:02,076 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/bucket.cache 2024-11-28T01:45:02,076 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-28T01:45:02,076 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-28T01:45:02,076 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-28T01:45:02,080 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-28T01:45:02,080 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-28T01:45:02,081 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-28T01:45:02,081 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/bucket.persistence is complete. 2024-11-28T01:45:02,083 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-28T01:45:02,084 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T01:45:02,085 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T01:45:02,088 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T01:45:02,093 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32859 2024-11-28T01:45:02,093 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32859 2024-11-28T01:45:02,095 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32859 2024-11-28T01:45:02,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32859 2024-11-28T01:45:02,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32859 2024-11-28T01:45:02,122 DEBUG [M:0;9df3f25f5867:40245 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9df3f25f5867:40245 2024-11-28T01:45:02,123 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9df3f25f5867,40245,1732758301060 2024-11-28T01:45:02,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T01:45:02,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T01:45:02,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T01:45:02,136 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9df3f25f5867,40245,1732758301060 2024-11-28T01:45:02,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T01:45:02,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T01:45:02,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:02,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:02,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:02,176 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T01:45:02,178 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9df3f25f5867,40245,1732758301060 from backup master directory 2024-11-28T01:45:02,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9df3f25f5867,40245,1732758301060 2024-11-28T01:45:02,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T01:45:02,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T01:45:02,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T01:45:02,183 WARN [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T01:45:02,184 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9df3f25f5867,40245,1732758301060 2024-11-28T01:45:02,186 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-28T01:45:02,188 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-28T01:45:02,199 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/hbase.id] with ID: 191e7394-c153-4275-8884-41c640b1ef5d 2024-11-28T01:45:02,199 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/.tmp/hbase.id 2024-11-28T01:45:02,206 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/hbase.id] 2024-11-28T01:45:02,248 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:02,255 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-28T01:45:02,284 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-11-28T01:45:02,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:02,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:02,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:02,317 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T01:45:02,320 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-28T01:45:02,368 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:45:02,374 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T01:45:02,438 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store 2024-11-28T01:45:02,465 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-28T01:45:02,472 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:45:02,474 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T01:45:02,474 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:45:02,475 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:45:02,477 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T01:45:02,477 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:45:02,477 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:45:02,478 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732758302474Disabling compacts and flushes for region at 1732758302474Disabling writes for close at 1732758302477 (+3 ms)Writing region close event to WAL at 1732758302477Closed at 1732758302477 2024-11-28T01:45:02,479 WARN [master/9df3f25f5867:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/.initializing 2024-11-28T01:45:02,479 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/WALs/9df3f25f5867,40245,1732758301060 2024-11-28T01:45:02,497 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T01:45:02,515 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9df3f25f5867%2C40245%2C1732758301060, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/WALs/9df3f25f5867,40245,1732758301060, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/oldWALs, maxLogs=10 2024-11-28T01:45:02,565 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/WALs/9df3f25f5867,40245,1732758301060/9df3f25f5867%2C40245%2C1732758301060.1732758302521 2024-11-28T01:45:02,566 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T01:45:02,567 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:45:02,568 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:45:02,573 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:02,574 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:02,619 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:02,650 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-28T01:45:02,654 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:02,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:02,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:02,661 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-28T01:45:02,661 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:02,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:45:02,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:02,666 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-28T01:45:02,667 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:02,667 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:45:02,667 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:02,673 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-28T01:45:02,673 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:02,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:45:02,674 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:02,678 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:02,678 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:02,681 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:02,682 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:02,685 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T01:45:02,687 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:02,696 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T01:45:02,697 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70261615, jitterRate=0.04697965085506439}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T01:45:02,704 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732758302593Initializing all the Stores at 1732758302594 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758302594Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758302595 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758302596 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758302596Cleaning up temporary data from old regions at 1732758302682 (+86 ms)Region opened successfully at 1732758302704 (+22 ms) 2024-11-28T01:45:02,706 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-28T01:45:02,746 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e2b8aad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9df3f25f5867/172.17.0.2:0 2024-11-28T01:45:02,782 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-28T01:45:02,799 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-28T01:45:02,799 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-28T01:45:02,803 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T01:45:02,804 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T01:45:02,812 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-28T01:45:02,812 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-28T01:45:02,843 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-28T01:45:02,856 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-28T01:45:02,858 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-28T01:45:02,861 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-28T01:45:02,863 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-28T01:45:02,865 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-28T01:45:02,867 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-28T01:45:02,872 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-28T01:45:02,874 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-28T01:45:02,876 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-28T01:45:02,878 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-28T01:45:02,903 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-28T01:45:02,905 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-28T01:45:02,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T01:45:02,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T01:45:02,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:02,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T01:45:02,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:02,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:02,924 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9df3f25f5867,40245,1732758301060, sessionid=0x1003bcd212c0000, setting cluster-up flag (Was=false) 2024-11-28T01:45:02,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:02,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:02,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:02,946 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-28T01:45:02,948 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9df3f25f5867,40245,1732758301060 2024-11-28T01:45:02,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:02,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:02,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:02,959 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-28T01:45:02,961 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9df3f25f5867,40245,1732758301060 2024-11-28T01:45:02,967 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-28T01:45:03,002 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(746): ClusterId : 191e7394-c153-4275-8884-41c640b1ef5d 2024-11-28T01:45:03,002 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(746): ClusterId : 191e7394-c153-4275-8884-41c640b1ef5d 2024-11-28T01:45:03,006 DEBUG [RS:1;9df3f25f5867:32859 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T01:45:03,006 DEBUG [RS:0;9df3f25f5867:35035 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T01:45:03,012 DEBUG [RS:1;9df3f25f5867:32859 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T01:45:03,012 DEBUG [RS:0;9df3f25f5867:35035 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T01:45:03,012 DEBUG [RS:1;9df3f25f5867:32859 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T01:45:03,012 DEBUG [RS:0;9df3f25f5867:35035 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T01:45:03,016 DEBUG [RS:0;9df3f25f5867:35035 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T01:45:03,016 DEBUG [RS:1;9df3f25f5867:32859 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T01:45:03,017 DEBUG [RS:0;9df3f25f5867:35035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2db0edf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9df3f25f5867/172.17.0.2:0 2024-11-28T01:45:03,017 DEBUG [RS:1;9df3f25f5867:32859 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c0804fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9df3f25f5867/172.17.0.2:0 2024-11-28T01:45:03,036 DEBUG [RS:0;9df3f25f5867:35035 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9df3f25f5867:35035 2024-11-28T01:45:03,038 DEBUG [RS:1;9df3f25f5867:32859 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9df3f25f5867:32859 2024-11-28T01:45:03,040 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T01:45:03,040 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T01:45:03,040 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T01:45:03,040 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T01:45:03,040 DEBUG [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T01:45:03,040 DEBUG [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T01:45:03,044 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(2659): reportForDuty to master=9df3f25f5867,40245,1732758301060 with port=35035, startcode=1732758301902 2024-11-28T01:45:03,048 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(2659): reportForDuty to master=9df3f25f5867,40245,1732758301060 with port=32859, startcode=1732758302059 2024-11-28T01:45:03,058 DEBUG [RS:1;9df3f25f5867:32859 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T01:45:03,058 DEBUG [RS:0;9df3f25f5867:35035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T01:45:03,064 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T01:45:03,078 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-28T01:45:03,088 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-28T01:45:03,094 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9df3f25f5867,40245,1732758301060 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-28T01:45:03,104 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9df3f25f5867:0, corePoolSize=5, maxPoolSize=5 2024-11-28T01:45:03,104 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9df3f25f5867:0, corePoolSize=5, maxPoolSize=5 2024-11-28T01:45:03,104 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9df3f25f5867:0, corePoolSize=5, maxPoolSize=5 2024-11-28T01:45:03,107 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9df3f25f5867:0, corePoolSize=5, maxPoolSize=5 2024-11-28T01:45:03,109 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9df3f25f5867:0, corePoolSize=10, maxPoolSize=10 2024-11-28T01:45:03,112 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,112 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9df3f25f5867:0, corePoolSize=2, maxPoolSize=2 2024-11-28T01:45:03,113 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,132 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T01:45:03,132 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-28T01:45:03,138 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732758333138 2024-11-28T01:45:03,140 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59203, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T01:45:03,140 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46917, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T01:45:03,140 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-28T01:45:03,141 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:03,141 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-28T01:45:03,142 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-28T01:45:03,146 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-28T01:45:03,146 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-28T01:45:03,147 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-28T01:45:03,147 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-28T01:45:03,149 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40245 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:45:03,150 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,154 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-28T01:45:03,156 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-28T01:45:03,156 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-28T01:45:03,157 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40245 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T01:45:03,164 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-28T01:45:03,164 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-28T01:45:03,165 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-28T01:45:03,165 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c 2024-11-28T01:45:03,166 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.large.0-1732758303166,5,FailOnTimeoutGroup] 2024-11-28T01:45:03,167 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.small.0-1732758303167,5,FailOnTimeoutGroup] 2024-11-28T01:45:03,167 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,167 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-28T01:45:03,169 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,169 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:45:03,179 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-28T01:45:03,185 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T01:45:03,185 DEBUG [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-28T01:45:03,185 DEBUG [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-28T01:45:03,186 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:03,186 WARN [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-28T01:45:03,186 WARN [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-28T01:45:03,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:03,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-28T01:45:03,199 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T01:45:03,199 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:03,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:03,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-28T01:45:03,204 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T01:45:03,205 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:03,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:03,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-28T01:45:03,209 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T01:45:03,209 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:03,210 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:03,210 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T01:45:03,211 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740 2024-11-28T01:45:03,212 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740 2024-11-28T01:45:03,212 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T01:45:03,213 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T01:45:03,213 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T01:45:03,214 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T01:45:03,221 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T01:45:03,221 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72968787, jitterRate=0.08731965720653534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T01:45:03,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732758303178Initializing all the Stores at 1732758303178Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758303178Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758303178Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758303180 (+2 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758303180Cleaning up temporary data from old regions at 1732758303213 (+33 ms)Region opened successfully at 1732758303224 (+11 ms) 2024-11-28T01:45:03,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T01:45:03,225 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T01:45:03,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T01:45:03,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T01:45:03,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T01:45:03,227 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T01:45:03,227 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732758303224Disabling compacts and flushes for region at 1732758303224Disabling writes for close at 1732758303225 (+1 ms)Writing region close event to WAL at 1732758303226 (+1 ms)Closed at 1732758303227 (+1 ms) 2024-11-28T01:45:03,230 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T01:45:03,230 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-28T01:45:03,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T01:45:03,247 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T01:45:03,249 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-28T01:45:03,287 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(2659): reportForDuty to master=9df3f25f5867,40245,1732758301060 with port=32859, startcode=1732758302059 2024-11-28T01:45:03,287 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(2659): reportForDuty to master=9df3f25f5867,40245,1732758301060 with port=35035, startcode=1732758301902 2024-11-28T01:45:03,291 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40245 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9df3f25f5867,32859,1732758302059 2024-11-28T01:45:03,294 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40245 {}] master.ServerManager(517): Registering regionserver=9df3f25f5867,32859,1732758302059 2024-11-28T01:45:03,302 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40245 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9df3f25f5867,35035,1732758301902 2024-11-28T01:45:03,302 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40245 {}] master.ServerManager(517): Registering regionserver=9df3f25f5867,35035,1732758301902 2024-11-28T01:45:03,302 DEBUG [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c 2024-11-28T01:45:03,303 DEBUG [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-28T01:45:03,303 DEBUG [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T01:45:03,305 DEBUG [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c 2024-11-28T01:45:03,305 DEBUG [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-28T01:45:03,305 DEBUG [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T01:45:03,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T01:45:03,313 DEBUG [RS:0;9df3f25f5867:35035 {}] zookeeper.ZKUtil(111): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9df3f25f5867,35035,1732758301902 2024-11-28T01:45:03,314 DEBUG [RS:1;9df3f25f5867:32859 {}] zookeeper.ZKUtil(111): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9df3f25f5867,32859,1732758302059 2024-11-28T01:45:03,314 WARN [RS:0;9df3f25f5867:35035 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T01:45:03,314 WARN [RS:1;9df3f25f5867:32859 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T01:45:03,314 INFO [RS:0;9df3f25f5867:35035 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T01:45:03,314 INFO [RS:1;9df3f25f5867:32859 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T01:45:03,314 DEBUG [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,35035,1732758301902 2024-11-28T01:45:03,314 DEBUG [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,32859,1732758302059 2024-11-28T01:45:03,317 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9df3f25f5867,35035,1732758301902] 2024-11-28T01:45:03,317 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9df3f25f5867,32859,1732758302059] 2024-11-28T01:45:03,341 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T01:45:03,341 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T01:45:03,356 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T01:45:03,356 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T01:45:03,362 INFO [RS:1;9df3f25f5867:32859 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T01:45:03,362 INFO [RS:0;9df3f25f5867:35035 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T01:45:03,362 INFO [RS:1;9df3f25f5867:32859 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,362 INFO [RS:0;9df3f25f5867:35035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,364 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T01:45:03,364 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T01:45:03,369 INFO [RS:0;9df3f25f5867:35035 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T01:45:03,369 INFO [RS:1;9df3f25f5867:32859 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T01:45:03,371 INFO [RS:1;9df3f25f5867:32859 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,371 INFO [RS:0;9df3f25f5867:35035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,371 DEBUG [RS:1;9df3f25f5867:32859 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,371 DEBUG [RS:0;9df3f25f5867:35035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,371 DEBUG [RS:1;9df3f25f5867:32859 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,371 DEBUG [RS:0;9df3f25f5867:35035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,372 DEBUG [RS:1;9df3f25f5867:32859 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,372 DEBUG [RS:0;9df3f25f5867:35035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,372 DEBUG [RS:1;9df3f25f5867:32859 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,372 DEBUG [RS:1;9df3f25f5867:32859 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,372 DEBUG [RS:0;9df3f25f5867:35035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,372 DEBUG [RS:1;9df3f25f5867:32859 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9df3f25f5867:0, corePoolSize=2, maxPoolSize=2 2024-11-28T01:45:03,372 DEBUG [RS:0;9df3f25f5867:35035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,372 DEBUG [RS:1;9df3f25f5867:32859 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,372 DEBUG [RS:1;9df3f25f5867:32859 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,372 DEBUG [RS:0;9df3f25f5867:35035 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9df3f25f5867:0, corePoolSize=2, maxPoolSize=2 2024-11-28T01:45:03,373 DEBUG [RS:1;9df3f25f5867:32859 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,373 DEBUG [RS:1;9df3f25f5867:32859 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,373 DEBUG [RS:0;9df3f25f5867:35035 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,373 DEBUG [RS:1;9df3f25f5867:32859 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,373 DEBUG [RS:0;9df3f25f5867:35035 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,373 DEBUG [RS:1;9df3f25f5867:32859 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,373 DEBUG [RS:1;9df3f25f5867:32859 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9df3f25f5867:0, corePoolSize=3, maxPoolSize=3 2024-11-28T01:45:03,373 DEBUG [RS:0;9df3f25f5867:35035 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,373 DEBUG [RS:1;9df3f25f5867:32859 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0, corePoolSize=3, maxPoolSize=3 2024-11-28T01:45:03,374 DEBUG [RS:0;9df3f25f5867:35035 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,374 DEBUG [RS:0;9df3f25f5867:35035 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,374 DEBUG [RS:0;9df3f25f5867:35035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:03,374 DEBUG [RS:0;9df3f25f5867:35035 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9df3f25f5867:0, corePoolSize=3, maxPoolSize=3 2024-11-28T01:45:03,375 DEBUG [RS:0;9df3f25f5867:35035 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0, corePoolSize=3, maxPoolSize=3 2024-11-28T01:45:03,375 INFO [RS:1;9df3f25f5867:32859 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,375 INFO [RS:1;9df3f25f5867:32859 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,375 INFO [RS:1;9df3f25f5867:32859 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,376 INFO [RS:1;9df3f25f5867:32859 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,376 INFO [RS:0;9df3f25f5867:35035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,376 INFO [RS:1;9df3f25f5867:32859 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,376 INFO [RS:0;9df3f25f5867:35035 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,376 INFO [RS:1;9df3f25f5867:32859 {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,32859,1732758302059-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T01:45:03,376 INFO [RS:0;9df3f25f5867:35035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,376 INFO [RS:0;9df3f25f5867:35035 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,376 INFO [RS:0;9df3f25f5867:35035 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,376 INFO [RS:0;9df3f25f5867:35035 {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,35035,1732758301902-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T01:45:03,399 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T01:45:03,400 WARN [9df3f25f5867:40245 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T01:45:03,402 INFO [RS:1;9df3f25f5867:32859 {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,32859,1732758302059-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,402 INFO [RS:1;9df3f25f5867:32859 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,402 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.Replication(171): 9df3f25f5867,32859,1732758302059 started 2024-11-28T01:45:03,408 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T01:45:03,408 INFO [RS:0;9df3f25f5867:35035 {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,35035,1732758301902-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,408 INFO [RS:0;9df3f25f5867:35035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,408 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.Replication(171): 9df3f25f5867,35035,1732758301902 started 2024-11-28T01:45:03,423 INFO [RS:1;9df3f25f5867:32859 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,423 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(1482): Serving as 9df3f25f5867,32859,1732758302059, RpcServer on 9df3f25f5867/172.17.0.2:32859, sessionid=0x1003bcd212c0002 2024-11-28T01:45:03,424 DEBUG [RS:1;9df3f25f5867:32859 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T01:45:03,424 DEBUG [RS:1;9df3f25f5867:32859 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9df3f25f5867,32859,1732758302059 2024-11-28T01:45:03,424 DEBUG [RS:1;9df3f25f5867:32859 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9df3f25f5867,32859,1732758302059' 2024-11-28T01:45:03,424 DEBUG [RS:1;9df3f25f5867:32859 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T01:45:03,426 DEBUG [RS:1;9df3f25f5867:32859 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T01:45:03,426 DEBUG [RS:1;9df3f25f5867:32859 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T01:45:03,426 DEBUG [RS:1;9df3f25f5867:32859 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T01:45:03,427 DEBUG [RS:1;9df3f25f5867:32859 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9df3f25f5867,32859,1732758302059 2024-11-28T01:45:03,427 DEBUG [RS:1;9df3f25f5867:32859 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9df3f25f5867,32859,1732758302059' 2024-11-28T01:45:03,427 DEBUG [RS:1;9df3f25f5867:32859 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T01:45:03,427 DEBUG [RS:1;9df3f25f5867:32859 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T01:45:03,428 DEBUG [RS:1;9df3f25f5867:32859 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T01:45:03,428 INFO [RS:1;9df3f25f5867:32859 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T01:45:03,428 INFO [RS:1;9df3f25f5867:32859 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T01:45:03,433 INFO [RS:0;9df3f25f5867:35035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:03,433 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(1482): Serving as 9df3f25f5867,35035,1732758301902, RpcServer on 9df3f25f5867/172.17.0.2:35035, sessionid=0x1003bcd212c0001 2024-11-28T01:45:03,433 DEBUG [RS:0;9df3f25f5867:35035 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T01:45:03,433 DEBUG [RS:0;9df3f25f5867:35035 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9df3f25f5867,35035,1732758301902 2024-11-28T01:45:03,433 DEBUG [RS:0;9df3f25f5867:35035 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9df3f25f5867,35035,1732758301902' 2024-11-28T01:45:03,434 DEBUG [RS:0;9df3f25f5867:35035 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T01:45:03,434 DEBUG [RS:0;9df3f25f5867:35035 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T01:45:03,435 DEBUG [RS:0;9df3f25f5867:35035 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T01:45:03,435 DEBUG [RS:0;9df3f25f5867:35035 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T01:45:03,435 DEBUG [RS:0;9df3f25f5867:35035 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9df3f25f5867,35035,1732758301902 2024-11-28T01:45:03,435 DEBUG [RS:0;9df3f25f5867:35035 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9df3f25f5867,35035,1732758301902' 2024-11-28T01:45:03,435 DEBUG [RS:0;9df3f25f5867:35035 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T01:45:03,436 DEBUG [RS:0;9df3f25f5867:35035 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T01:45:03,437 DEBUG [RS:0;9df3f25f5867:35035 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T01:45:03,437 INFO [RS:0;9df3f25f5867:35035 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T01:45:03,437 INFO [RS:0;9df3f25f5867:35035 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T01:45:03,534 INFO [RS:1;9df3f25f5867:32859 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T01:45:03,535 INFO [RS:1;9df3f25f5867:32859 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9df3f25f5867%2C32859%2C1732758302059, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,32859,1732758302059, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/oldWALs, maxLogs=55 2024-11-28T01:45:03,538 INFO [RS:0;9df3f25f5867:35035 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T01:45:03,539 INFO [RS:0;9df3f25f5867:35035 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9df3f25f5867%2C35035%2C1732758301902, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,35035,1732758301902, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/oldWALs, maxLogs=55 2024-11-28T01:45:03,557 INFO [RS:1;9df3f25f5867:32859 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,32859,1732758302059/9df3f25f5867%2C32859%2C1732758302059.1732758303537 2024-11-28T01:45:03,558 DEBUG [RS:1;9df3f25f5867:32859 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T01:45:03,568 INFO [RS:0;9df3f25f5867:35035 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,35035,1732758301902/9df3f25f5867%2C35035%2C1732758301902.1732758303542 2024-11-28T01:45:03,569 DEBUG [RS:0;9df3f25f5867:35035 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T01:45:03,653 DEBUG [9df3f25f5867:40245 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-28T01:45:03,665 DEBUG [9df3f25f5867:40245 {}] balancer.BalancerClusterState(204): Hosts are {9df3f25f5867=0} racks are {/default-rack=0} 2024-11-28T01:45:03,669 DEBUG [9df3f25f5867:40245 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T01:45:03,669 DEBUG [9df3f25f5867:40245 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T01:45:03,669 DEBUG [9df3f25f5867:40245 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T01:45:03,669 DEBUG [9df3f25f5867:40245 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T01:45:03,669 INFO [9df3f25f5867:40245 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T01:45:03,669 INFO [9df3f25f5867:40245 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T01:45:03,669 DEBUG [9df3f25f5867:40245 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T01:45:03,677 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9df3f25f5867,32859,1732758302059 2024-11-28T01:45:03,684 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9df3f25f5867,32859,1732758302059, state=OPENING 2024-11-28T01:45:03,690 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-28T01:45:03,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:03,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:03,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:03,693 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:03,693 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:03,693 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:03,694 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T01:45:03,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9df3f25f5867,32859,1732758302059}] 2024-11-28T01:45:03,870 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T01:45:03,873 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32775, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T01:45:03,883 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T01:45:03,884 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T01:45:03,884 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T01:45:03,886 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9df3f25f5867%2C32859%2C1732758302059.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,32859,1732758302059, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/oldWALs, maxLogs=55 2024-11-28T01:45:03,907 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,32859,1732758302059/9df3f25f5867%2C32859%2C1732758302059.meta.1732758303887.meta 2024-11-28T01:45:03,908 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T01:45:03,908 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:45:03,910 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T01:45:03,913 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T01:45:03,918 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T01:45:03,922 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T01:45:03,923 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:45:03,923 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T01:45:03,923 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T01:45:03,925 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-28T01:45:03,926 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T01:45:03,926 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:03,926 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:03,926 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-28T01:45:03,927 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T01:45:03,927 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:03,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:03,928 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-28T01:45:03,928 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T01:45:03,928 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:03,929 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:03,929 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-28T01:45:03,929 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T01:45:03,930 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:03,930 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:03,930 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T01:45:03,931 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740 2024-11-28T01:45:03,932 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740 2024-11-28T01:45:03,932 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T01:45:03,932 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T01:45:03,933 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T01:45:03,934 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T01:45:03,935 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60313667, jitterRate=-0.10125632584095001}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T01:45:03,935 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T01:45:03,936 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732758303924Writing region info on filesystem at 1732758303924Initializing all the Stores at 1732758303924Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758303924Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758303924Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758303924Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758303924Cleaning up temporary data from old regions at 1732758303933 (+9 ms)Running coprocessor post-open hooks at 1732758303935 (+2 ms)Region opened successfully at 1732758303936 (+1 ms) 2024-11-28T01:45:03,942 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732758303861 2024-11-28T01:45:03,953 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T01:45:03,953 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T01:45:03,954 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9df3f25f5867,32859,1732758302059 2024-11-28T01:45:03,955 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9df3f25f5867,32859,1732758302059, state=OPEN 2024-11-28T01:45:03,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T01:45:03,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T01:45:03,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T01:45:03,958 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:03,958 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:03,958 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:03,959 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9df3f25f5867,32859,1732758302059 2024-11-28T01:45:03,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-28T01:45:03,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9df3f25f5867,32859,1732758302059 in 263 msec 2024-11-28T01:45:03,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-28T01:45:03,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 725 msec 2024-11-28T01:45:03,967 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T01:45:03,967 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-28T01:45:03,988 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:45:03,989 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,32859,1732758302059, seqNum=-1] 2024-11-28T01:45:04,011 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:45:04,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38839, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:45:04,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0240 sec 2024-11-28T01:45:04,031 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732758304031, completionTime=-1 2024-11-28T01:45:04,034 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-28T01:45:04,034 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T01:45:04,064 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T01:45:04,064 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732758364064 2024-11-28T01:45:04,064 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732758424064 2024-11-28T01:45:04,064 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-11-28T01:45:04,068 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-28T01:45:04,077 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,40245,1732758301060-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:04,078 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,40245,1732758301060-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:04,078 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,40245,1732758301060-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:04,080 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9df3f25f5867:40245, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:04,080 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:04,081 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:04,087 DEBUG [master/9df3f25f5867:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-28T01:45:04,118 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.934sec 2024-11-28T01:45:04,120 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-28T01:45:04,122 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-28T01:45:04,123 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-28T01:45:04,123 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-28T01:45:04,124 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-28T01:45:04,125 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,40245,1732758301060-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T01:45:04,125 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,40245,1732758301060-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-28T01:45:04,127 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-28T01:45:04,128 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-28T01:45:04,128 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,40245,1732758301060-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:04,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32be4008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:45:04,216 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-28T01:45:04,217 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-28T01:45:04,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,40245,-1 for getting cluster id 2024-11-28T01:45:04,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:45:04,232 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '191e7394-c153-4275-8884-41c640b1ef5d' 2024-11-28T01:45:04,234 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:45:04,234 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "191e7394-c153-4275-8884-41c640b1ef5d" 2024-11-28T01:45:04,235 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c05388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:45:04,235 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,40245,-1] 2024-11-28T01:45:04,237 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:45:04,239 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:04,241 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:45:04,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3302ae21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:45:04,244 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:45:04,251 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,32859,1732758302059, seqNum=-1] 2024-11-28T01:45:04,252 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:45:04,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:45:04,289 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9df3f25f5867,40245,1732758301060 2024-11-28T01:45:04,295 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T01:45:04,304 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9df3f25f5867,40245,1732758301060 2024-11-28T01:45:04,307 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@67a44e26 2024-11-28T01:45:04,309 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T01:45:04,311 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T01:45:04,317 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40245 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732758304289', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T01:45:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40245 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732758304289 2024-11-28T01:45:04,327 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732758304289 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T01:45:04,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40245 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732758304289" procId is: 4 2024-11-28T01:45:04,329 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:04,330 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732758304289 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T01:45:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T01:45:04,345 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732758304289-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b0f7e501da2dbd7bc0b9225c14c05dcf, NAME => 'testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732758304289', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c 2024-11-28T01:45:04,352 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732758304289-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:45:04,352 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732758304289-pool-0 {}] regionserver.HRegion(1722): Closing b0f7e501da2dbd7bc0b9225c14c05dcf, disabling compactions & flushes 2024-11-28T01:45:04,352 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732758304289-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:04,352 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732758304289-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:04,352 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732758304289-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. after waiting 0 ms 2024-11-28T01:45:04,352 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732758304289-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:04,352 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732758304289-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:04,352 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732758304289-pool-0 {}] regionserver.HRegion(1676): Region close journal for b0f7e501da2dbd7bc0b9225c14c05dcf: Waiting for close lock at 1732758304352Disabling compacts and flushes for region at 1732758304352Disabling writes for close at 1732758304352Writing region close event to WAL at 1732758304352Closed at 1732758304352 2024-11-28T01:45:04,354 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732758304289 execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T01:45:04,359 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732758304354"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732758304354"}]},"ts":"1732758304354"} 2024-11-28T01:45:04,363 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T01:45:04,364 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732758304289 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T01:45:04,367 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732758304289","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758304364"}]},"ts":"1732758304364"} 2024-11-28T01:45:04,370 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732758304289, state=ENABLING in hbase:meta 2024-11-28T01:45:04,371 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {9df3f25f5867=0} racks are {/default-rack=0} 2024-11-28T01:45:04,371 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T01:45:04,371 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T01:45:04,371 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T01:45:04,371 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T01:45:04,371 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T01:45:04,371 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T01:45:04,372 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T01:45:04,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732758304289, region=b0f7e501da2dbd7bc0b9225c14c05dcf, ASSIGN}] 2024-11-28T01:45:04,374 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732758304289, region=b0f7e501da2dbd7bc0b9225c14c05dcf, ASSIGN 2024-11-28T01:45:04,375 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732758304289, region=b0f7e501da2dbd7bc0b9225c14c05dcf, ASSIGN; state=OFFLINE, location=9df3f25f5867,32859,1732758302059; forceNewPlan=false, retain=false 2024-11-28T01:45:04,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T01:45:04,528 INFO [9df3f25f5867:40245 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-28T01:45:04,529 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b0f7e501da2dbd7bc0b9225c14c05dcf, regionState=OPENING, regionLocation=9df3f25f5867,32859,1732758302059 2024-11-28T01:45:04,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732758304289, region=b0f7e501da2dbd7bc0b9225c14c05dcf, ASSIGN because future has completed 2024-11-28T01:45:04,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b0f7e501da2dbd7bc0b9225c14c05dcf, server=9df3f25f5867,32859,1732758302059}] 2024-11-28T01:45:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T01:45:04,691 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:04,692 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b0f7e501da2dbd7bc0b9225c14c05dcf, NAME => 'testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:45:04,693 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732758304289 b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:04,693 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:45:04,693 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:04,693 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:04,694 INFO [StoreOpener-b0f7e501da2dbd7bc0b9225c14c05dcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:04,699 INFO [StoreOpener-b0f7e501da2dbd7bc0b9225c14c05dcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b0f7e501da2dbd7bc0b9225c14c05dcf columnFamilyName family 2024-11-28T01:45:04,700 DEBUG [StoreOpener-b0f7e501da2dbd7bc0b9225c14c05dcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:04,701 INFO [StoreOpener-b0f7e501da2dbd7bc0b9225c14c05dcf-1 {}] regionserver.HStore(327): Store=b0f7e501da2dbd7bc0b9225c14c05dcf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:45:04,701 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:04,702 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:04,702 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:04,702 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:04,702 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:04,703 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:04,712 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T01:45:04,712 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b0f7e501da2dbd7bc0b9225c14c05dcf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70134474, jitterRate=0.04508510231971741}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T01:45:04,713 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:04,714 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b0f7e501da2dbd7bc0b9225c14c05dcf: Running coprocessor pre-open hook at 1732758304693Writing region info on filesystem at 1732758304693Initializing all the Stores at 1732758304694 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758304694Cleaning up temporary data from old regions at 1732758304702 (+8 ms)Running coprocessor post-open hooks at 1732758304713 (+11 ms)Region opened successfully at 1732758304714 (+1 ms) 2024-11-28T01:45:04,715 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf., pid=6, masterSystemTime=1732758304686 2024-11-28T01:45:04,719 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:04,719 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:04,720 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b0f7e501da2dbd7bc0b9225c14c05dcf, regionState=OPEN, openSeqNum=2, regionLocation=9df3f25f5867,32859,1732758302059 2024-11-28T01:45:04,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b0f7e501da2dbd7bc0b9225c14c05dcf, server=9df3f25f5867,32859,1732758302059 because future has completed 2024-11-28T01:45:04,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-28T01:45:04,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b0f7e501da2dbd7bc0b9225c14c05dcf, server=9df3f25f5867,32859,1732758302059 in 192 msec 2024-11-28T01:45:04,730 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-28T01:45:04,730 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732758304289, region=b0f7e501da2dbd7bc0b9225c14c05dcf, ASSIGN in 355 msec 2024-11-28T01:45:04,731 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732758304289 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T01:45:04,731 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732758304289","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758304731"}]},"ts":"1732758304731"} 2024-11-28T01:45:04,738 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732758304289, state=ENABLED in hbase:meta 2024-11-28T01:45:04,739 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732758304289 execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T01:45:04,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732758304289 in 418 msec 2024-11-28T01:45:04,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40245 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T01:45:04,959 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732758304289 completed 2024-11-28T01:45:04,959 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732758304289 get assigned. Timeout = 60000ms 2024-11-28T01:45:04,963 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T01:45:04,968 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732758304289 assigned to meta. Checking AM states. 2024-11-28T01:45:04,969 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T01:45:04,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732758304289 assigned. 2024-11-28T01:45:04,977 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732758304289', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf., hostname=9df3f25f5867,32859,1732758302059, seqNum=2] 2024-11-28T01:45:04,989 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b0f7e501da2dbd7bc0b9225c14c05dcf 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-28T01:45:05,017 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d11ec217b35240e2926d2d2119b1b97d 2024-11-28T01:45:05,057 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/.tmp/family/d11ec217b35240e2926d2d2119b1b97d is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-28T01:45:05,063 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/.tmp/family/d11ec217b35240e2926d2d2119b1b97d 2024-11-28T01:45:05,100 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/.tmp/family/d11ec217b35240e2926d2d2119b1b97d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/family/d11ec217b35240e2926d2d2119b1b97d 2024-11-28T01:45:05,102 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/family/d11ec217b35240e2926d2d2119b1b97d, delay=974 ms 2024-11-28T01:45:05,103 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/family/d11ec217b35240e2926d2d2119b1b97d, entries=2, sequenceid=6, filesize=4.9 K 2024-11-28T01:45:05,109 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for b0f7e501da2dbd7bc0b9225c14c05dcf in 121ms, sequenceid=6, compaction requested=false 2024-11-28T01:45:05,110 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732758304289' 2024-11-28T01:45:05,113 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b0f7e501da2dbd7bc0b9225c14c05dcf: 2024-11-28T01:45:06,078 DEBUG [hfile-prefetch-1732758305102 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d11ec217b35240e2926d2d2119b1b97d_0. 2024-11-28T01:45:06,083 DEBUG [hfile-prefetch-1732758305102 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d11ec217b35240e2926d2d2119b1b97d_117. 2024-11-28T01:45:06,084 DEBUG [hfile-prefetch-1732758305102 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/family/d11ec217b35240e2926d2d2119b1b97d, with total blocks 2, and data blocks 1 2024-11-28T01:45:06,084 DEBUG [hfile-prefetch-1732758305102 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-28T01:45:06,085 DEBUG [hfile-prefetch-1732758305102 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/family/d11ec217b35240e2926d2d2119b1b97d, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-28T01:45:06,085 DEBUG [hfile-prefetch-1732758305102 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/family/d11ec217b35240e2926d2d2119b1b97d. Will sleep 100ms and try the verification again. 2024-11-28T01:45:06,102 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-28T01:45:06,102 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-28T01:45:06,102 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T01:45:06,102 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T01:45:06,120 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-28T01:45:06,121 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732758306102 2024-11-28T01:45:06,121 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-28T01:45:06,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40245 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=b0f7e501da2dbd7bc0b9225c14c05dcf, source=9df3f25f5867,32859,1732758302059, destination=9df3f25f5867,35035,1732758301902, warming up region on 9df3f25f5867,35035,1732758301902 2024-11-28T01:45:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40245 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T01:45:06,128 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40245 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=b0f7e501da2dbd7bc0b9225c14c05dcf, source=9df3f25f5867,32859,1732758302059, destination=9df3f25f5867,35035,1732758301902, running balancer 2024-11-28T01:45:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40245 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732758304289, region=b0f7e501da2dbd7bc0b9225c14c05dcf, REOPEN/MOVE 2024-11-28T01:45:06,130 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732758304289, region=b0f7e501da2dbd7bc0b9225c14c05dcf, REOPEN/MOVE 2024-11-28T01:45:06,130 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36343, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T01:45:06,132 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b0f7e501da2dbd7bc0b9225c14c05dcf, regionState=CLOSING, regionLocation=9df3f25f5867,32859,1732758302059 2024-11-28T01:45:06,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35035 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35035 {}] regionserver.HRegion(7855): Warmup {ENCODED => b0f7e501da2dbd7bc0b9225c14c05dcf, NAME => 'testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:45:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35035 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:45:06,134 INFO [StoreOpener-b0f7e501da2dbd7bc0b9225c14c05dcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:06,135 INFO [StoreOpener-b0f7e501da2dbd7bc0b9225c14c05dcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b0f7e501da2dbd7bc0b9225c14c05dcf columnFamilyName family 2024-11-28T01:45:06,135 DEBUG [StoreOpener-b0f7e501da2dbd7bc0b9225c14c05dcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:06,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732758304289, region=b0f7e501da2dbd7bc0b9225c14c05dcf, REOPEN/MOVE because future has completed 2024-11-28T01:45:06,136 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-28T01:45:06,136 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure b0f7e501da2dbd7bc0b9225c14c05dcf, server=9df3f25f5867,32859,1732758302059}] 2024-11-28T01:45:06,143 DEBUG [StoreOpener-b0f7e501da2dbd7bc0b9225c14c05dcf-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/family/d11ec217b35240e2926d2d2119b1b97d 2024-11-28T01:45:06,143 INFO [StoreOpener-b0f7e501da2dbd7bc0b9225c14c05dcf-1 {}] regionserver.HStore(327): Store=b0f7e501da2dbd7bc0b9225c14c05dcf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:45:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35035 {}] regionserver.HRegion(1722): Closing b0f7e501da2dbd7bc0b9225c14c05dcf, disabling compactions & flushes 2024-11-28T01:45:06,144 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35035 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35035 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35035 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. after waiting 0 ms 2024-11-28T01:45:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35035 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,145 DEBUG [StoreFileCloser-b0f7e501da2dbd7bc0b9225c14c05dcf-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/family/d11ec217b35240e2926d2d2119b1b97d 2024-11-28T01:45:06,145 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35035 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35035 {}] regionserver.HRegion(1676): Region close journal for b0f7e501da2dbd7bc0b9225c14c05dcf: Waiting for close lock at 1732758306143Disabling compacts and flushes for region at 1732758306143Disabling writes for close at 1732758306144 (+1 ms)Writing region close event to WAL at 1732758306145 (+1 ms)Closed at 1732758306145 2024-11-28T01:45:06,148 WARN [hfile-prefetch-1732758305102 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/family/d11ec217b35240e2926d2d2119b1b97d, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-28T01:45:06,148 DEBUG [hfile-prefetch-1732758305102 {}] hfile.PrefetchExecutor(121): Prefetch completed for d11ec217b35240e2926d2d2119b1b97d 2024-11-28T01:45:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40245 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-28T01:45:06,294 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:06,294 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-28T01:45:06,295 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing b0f7e501da2dbd7bc0b9225c14c05dcf, disabling compactions & flushes 2024-11-28T01:45:06,295 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,295 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,295 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. after waiting 0 ms 2024-11-28T01:45:06,295 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,297 DEBUG [StoreFileCloser-b0f7e501da2dbd7bc0b9225c14c05dcf-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file d11ec217b35240e2926d2d2119b1b97d, starting offset: 0, end offset: 9223372036854775807 2024-11-28T01:45:06,299 DEBUG [StoreFileCloser-b0f7e501da2dbd7bc0b9225c14c05dcf-family-1 {}] bucket.BucketCache(856): removed key d11ec217b35240e2926d2d2119b1b97d_0 from back map with offset lock 2094080 in the evict process 2024-11-28T01:45:06,304 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-28T01:45:06,305 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,306 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for b0f7e501da2dbd7bc0b9225c14c05dcf: Waiting for close lock at 1732758306295Running coprocessor pre-close hooks at 1732758306295Disabling compacts and flushes for region at 1732758306295Disabling writes for close at 1732758306295Writing region close event to WAL at 1732758306299 (+4 ms)Running coprocessor post-close hooks at 1732758306304 (+5 ms)Closed at 1732758306305 (+1 ms) 2024-11-28T01:45:06,306 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding b0f7e501da2dbd7bc0b9225c14c05dcf move to 9df3f25f5867,35035,1732758301902 record at close sequenceid=6 2024-11-28T01:45:06,309 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:06,310 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b0f7e501da2dbd7bc0b9225c14c05dcf, regionState=CLOSED 2024-11-28T01:45:06,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure b0f7e501da2dbd7bc0b9225c14c05dcf, server=9df3f25f5867,32859,1732758302059 because future has completed 2024-11-28T01:45:06,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-28T01:45:06,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure b0f7e501da2dbd7bc0b9225c14c05dcf, server=9df3f25f5867,32859,1732758302059 in 178 msec 2024-11-28T01:45:06,316 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732758304289, region=b0f7e501da2dbd7bc0b9225c14c05dcf, REOPEN/MOVE; state=CLOSED, location=9df3f25f5867,35035,1732758301902; forceNewPlan=false, retain=false 2024-11-28T01:45:06,321 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-28T01:45:06,321 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-28T01:45:06,321 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T01:45:06,321 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T01:45:06,325 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-28T01:45:06,325 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732758306321 2024-11-28T01:45:06,325 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-28T01:45:06,467 INFO [9df3f25f5867:40245 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-28T01:45:06,467 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b0f7e501da2dbd7bc0b9225c14c05dcf, regionState=OPENING, regionLocation=9df3f25f5867,35035,1732758301902 2024-11-28T01:45:06,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732758304289, region=b0f7e501da2dbd7bc0b9225c14c05dcf, REOPEN/MOVE because future has completed 2024-11-28T01:45:06,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure b0f7e501da2dbd7bc0b9225c14c05dcf, server=9df3f25f5867,35035,1732758301902}] 2024-11-28T01:45:06,626 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,627 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => b0f7e501da2dbd7bc0b9225c14c05dcf, NAME => 'testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:45:06,627 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732758304289 b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:06,627 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:45:06,627 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:06,627 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:06,628 INFO [StoreOpener-b0f7e501da2dbd7bc0b9225c14c05dcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:06,630 INFO [StoreOpener-b0f7e501da2dbd7bc0b9225c14c05dcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b0f7e501da2dbd7bc0b9225c14c05dcf columnFamilyName family 2024-11-28T01:45:06,630 DEBUG [StoreOpener-b0f7e501da2dbd7bc0b9225c14c05dcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:06,632 DEBUG [StoreFileOpener-b0f7e501da2dbd7bc0b9225c14c05dcf-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/family/d11ec217b35240e2926d2d2119b1b97d, delay=956 ms 2024-11-28T01:45:06,633 DEBUG [StoreOpener-b0f7e501da2dbd7bc0b9225c14c05dcf-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/family/d11ec217b35240e2926d2d2119b1b97d 2024-11-28T01:45:06,633 INFO [StoreOpener-b0f7e501da2dbd7bc0b9225c14c05dcf-1 {}] regionserver.HStore(327): Store=b0f7e501da2dbd7bc0b9225c14c05dcf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:45:06,634 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:06,634 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:06,635 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:06,635 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:06,635 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:06,642 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:06,643 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened b0f7e501da2dbd7bc0b9225c14c05dcf; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63240515, jitterRate=-0.05764289200305939}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T01:45:06,643 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:06,644 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for b0f7e501da2dbd7bc0b9225c14c05dcf: Running coprocessor pre-open hook at 1732758306628Writing region info on filesystem at 1732758306628Initializing all the Stores at 1732758306628Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758306628Cleaning up temporary data from old regions at 1732758306635 (+7 ms)Running coprocessor post-open hooks at 1732758306643 (+8 ms)Region opened successfully at 1732758306644 (+1 ms) 2024-11-28T01:45:06,645 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf., pid=9, masterSystemTime=1732758306623 2024-11-28T01:45:06,649 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,649 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,649 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b0f7e501da2dbd7bc0b9225c14c05dcf, regionState=OPEN, openSeqNum=10, regionLocation=9df3f25f5867,35035,1732758301902 2024-11-28T01:45:06,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure b0f7e501da2dbd7bc0b9225c14c05dcf, server=9df3f25f5867,35035,1732758301902 because future has completed 2024-11-28T01:45:06,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-28T01:45:06,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure b0f7e501da2dbd7bc0b9225c14c05dcf, server=9df3f25f5867,35035,1732758301902 in 183 msec 2024-11-28T01:45:06,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732758304289, region=b0f7e501da2dbd7bc0b9225c14c05dcf, REOPEN/MOVE in 528 msec 2024-11-28T01:45:06,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-28T01:45:06,683 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T01:45:06,683 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:45:06,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:06,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:06,689 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:45:06,689 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-28T01:45:06,689 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-28T01:45:06,690 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9df3f25f5867,40245,1732758301060 2024-11-28T01:45:06,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T01:45:06,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:06,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T01:45:06,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T01:45:06,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:06,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:06,693 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T01:45:06,693 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T01:45:06,694 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T01:45:06,694 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:45:06,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:06,694 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T01:45:06,694 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T01:45:06,695 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9df3f25f5867,35035,1732758301902' ***** 2024-11-28T01:45:06,695 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T01:45:06,695 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9df3f25f5867,32859,1732758302059' ***** 2024-11-28T01:45:06,695 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T01:45:06,695 INFO [RS:0;9df3f25f5867:35035 {}] bucket.BucketCache(1738): Disabling cache 2024-11-28T01:45:06,695 INFO [RS:1;9df3f25f5867:32859 {}] bucket.BucketCache(1738): Disabling cache 2024-11-28T01:45:06,696 INFO [RS:0;9df3f25f5867:35035 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/bucket.persistence 2024-11-28T01:45:06,696 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-28T01:45:06,696 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-28T01:45:06,696 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-28T01:45:06,696 INFO [RS:1;9df3f25f5867:32859 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/bucket.persistence 2024-11-28T01:45:06,696 INFO [RS:1;9df3f25f5867:32859 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-28T01:45:06,696 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-28T01:45:06,696 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-28T01:45:06,696 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-28T01:45:06,696 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-28T01:45:06,696 INFO [RS:0;9df3f25f5867:35035 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-28T01:45:06,697 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-28T01:45:06,706 DEBUG [RS:1;9df3f25f5867:32859 {}] bucket.BucketCache(1487): Thread RS:1;9df3f25f5867:32859 started persisting bucket cache to file 2024-11-28T01:45:06,706 DEBUG [RS:1;9df3f25f5867:32859 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T01:45:06,706 DEBUG [RS:1;9df3f25f5867:32859 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T01:45:06,707 DEBUG [RS:0;9df3f25f5867:35035 {}] bucket.BucketCache(1487): Thread RS:0;9df3f25f5867:35035 started persisting bucket cache to file 2024-11-28T01:45:06,707 DEBUG [RS:0;9df3f25f5867:35035 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T01:45:06,707 DEBUG [RS:0;9df3f25f5867:35035 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T01:45:06,710 DEBUG [RS:1;9df3f25f5867:32859 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-28T01:45:06,710 DEBUG [RS:1;9df3f25f5867:32859 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732758306706 2024-11-28T01:45:06,710 DEBUG [RS:1;9df3f25f5867:32859 {}] bucket.BucketCache(1509): Thread RS:1;9df3f25f5867:32859 finished persisting bucket cache to file, renaming 2024-11-28T01:45:06,711 DEBUG [RS:0;9df3f25f5867:35035 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-28T01:45:06,711 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T01:45:06,711 DEBUG [RS:0;9df3f25f5867:35035 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732758306707 2024-11-28T01:45:06,711 DEBUG [RS:0;9df3f25f5867:35035 {}] bucket.BucketCache(1509): Thread RS:0;9df3f25f5867:35035 finished persisting bucket cache to file, renaming 2024-11-28T01:45:06,711 INFO [RS:1;9df3f25f5867:32859 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T01:45:06,711 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T01:45:06,711 INFO [RS:1;9df3f25f5867:32859 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T01:45:06,711 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(959): stopping server 9df3f25f5867,32859,1732758302059 2024-11-28T01:45:06,712 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T01:45:06,712 INFO [RS:1;9df3f25f5867:32859 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T01:45:06,712 INFO [RS:0;9df3f25f5867:35035 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T01:45:06,712 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T01:45:06,712 INFO [RS:1;9df3f25f5867:32859 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9df3f25f5867:32859. 2024-11-28T01:45:06,712 INFO [RS:0;9df3f25f5867:35035 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T01:45:06,712 DEBUG [RS:1;9df3f25f5867:32859 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:45:06,712 DEBUG [RS:1;9df3f25f5867:32859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:06,712 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(3091): Received CLOSE for b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:06,712 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T01:45:06,712 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T01:45:06,712 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T01:45:06,712 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T01:45:06,712 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(959): stopping server 9df3f25f5867,35035,1732758301902 2024-11-28T01:45:06,713 INFO [RS:0;9df3f25f5867:35035 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T01:45:06,713 INFO [RS:0;9df3f25f5867:35035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9df3f25f5867:35035. 2024-11-28T01:45:06,713 DEBUG [RS:0;9df3f25f5867:35035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:45:06,713 DEBUG [RS:0;9df3f25f5867:35035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:06,713 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T01:45:06,713 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-28T01:45:06,713 DEBUG [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(1325): Online Regions={b0f7e501da2dbd7bc0b9225c14c05dcf=testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf.} 2024-11-28T01:45:06,713 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b0f7e501da2dbd7bc0b9225c14c05dcf, disabling compactions & flushes 2024-11-28T01:45:06,713 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,714 DEBUG [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(1351): Waiting on b0f7e501da2dbd7bc0b9225c14c05dcf 2024-11-28T01:45:06,714 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,714 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. after waiting 0 ms 2024-11-28T01:45:06,714 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-28T01:45:06,714 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,714 DEBUG [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-28T01:45:06,714 DEBUG [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T01:45:06,714 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T01:45:06,714 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T01:45:06,714 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T01:45:06,714 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T01:45:06,714 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T01:45:06,714 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T01:45:06,714 DEBUG [StoreFileCloser-b0f7e501da2dbd7bc0b9225c14c05dcf-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/family/d11ec217b35240e2926d2d2119b1b97d 2024-11-28T01:45:06,715 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-28T01:45:06,720 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnRegionMove1732758304289/b0f7e501da2dbd7bc0b9225c14c05dcf/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-28T01:45:06,721 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,721 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b0f7e501da2dbd7bc0b9225c14c05dcf: Waiting for close lock at 1732758306713Running coprocessor pre-close hooks at 1732758306713Disabling compacts and flushes for region at 1732758306713Disabling writes for close at 1732758306714 (+1 ms)Writing region close event to WAL at 1732758306715 (+1 ms)Running coprocessor post-close hooks at 1732758306721 (+6 ms)Closed at 1732758306721 2024-11-28T01:45:06,721 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf. 2024-11-28T01:45:06,727 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a3f361dcf569422aa49970e356a3774a 2024-11-28T01:45:06,745 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/info/a3f361dcf569422aa49970e356a3774a is 205, key is testBlockEvictionOnRegionMove1732758304289,,1732758304313.b0f7e501da2dbd7bc0b9225c14c05dcf./info:regioninfo/1732758306649/Put/seqid=0 2024-11-28T01:45:06,746 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/info/a3f361dcf569422aa49970e356a3774a 2024-11-28T01:45:06,756 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c3c3a22b5d334d70b4e12c1812c9e6e5 2024-11-28T01:45:06,770 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/ns/c3c3a22b5d334d70b4e12c1812c9e6e5 is 43, key is default/ns:d/1732758304018/Put/seqid=0 2024-11-28T01:45:06,771 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/ns/c3c3a22b5d334d70b4e12c1812c9e6e5 2024-11-28T01:45:06,780 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c4c9c1e8187b42e49715ad2ccf5b2a92 2024-11-28T01:45:06,781 INFO [regionserver/9df3f25f5867:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T01:45:06,784 INFO [regionserver/9df3f25f5867:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T01:45:06,795 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/table/c4c9c1e8187b42e49715ad2ccf5b2a92 is 78, key is testBlockEvictionOnRegionMove1732758304289/table:state/1732758304731/Put/seqid=0 2024-11-28T01:45:06,795 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/table/c4c9c1e8187b42e49715ad2ccf5b2a92 2024-11-28T01:45:06,798 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/info/a3f361dcf569422aa49970e356a3774a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/info/a3f361dcf569422aa49970e356a3774a 2024-11-28T01:45:06,799 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/info/a3f361dcf569422aa49970e356a3774a, delay=981 ms 2024-11-28T01:45:06,801 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/info/a3f361dcf569422aa49970e356a3774a, entries=15, sequenceid=15, filesize=7.6 K 2024-11-28T01:45:06,801 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/ns/c3c3a22b5d334d70b4e12c1812c9e6e5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/ns/c3c3a22b5d334d70b4e12c1812c9e6e5 2024-11-28T01:45:06,803 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/ns/c3c3a22b5d334d70b4e12c1812c9e6e5, delay=941 ms 2024-11-28T01:45:06,804 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/ns/c3c3a22b5d334d70b4e12c1812c9e6e5, entries=2, sequenceid=15, filesize=5.0 K 2024-11-28T01:45:06,805 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/table/c4c9c1e8187b42e49715ad2ccf5b2a92 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/table/c4c9c1e8187b42e49715ad2ccf5b2a92 2024-11-28T01:45:06,806 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/table/c4c9c1e8187b42e49715ad2ccf5b2a92, delay=987 ms 2024-11-28T01:45:06,807 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/table/c4c9c1e8187b42e49715ad2ccf5b2a92, entries=2, sequenceid=15, filesize=5.3 K 2024-11-28T01:45:06,808 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=15, compaction requested=false 2024-11-28T01:45:06,808 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-28T01:45:06,809 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/info/a3f361dcf569422aa49970e356a3774a 2024-11-28T01:45:06,809 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/ns/c3c3a22b5d334d70b4e12c1812c9e6e5 2024-11-28T01:45:06,810 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/table/c4c9c1e8187b42e49715ad2ccf5b2a92 2024-11-28T01:45:06,815 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-28T01:45:06,815 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T01:45:06,816 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T01:45:06,816 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732758306714Running coprocessor pre-close hooks at 1732758306714Disabling compacts and flushes for region at 1732758306714Disabling writes for close at 1732758306714Obtaining lock to block concurrent updates at 1732758306715 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732758306715Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732758306715Flushing stores of hbase:meta,,1.1588230740 at 1732758306716 (+1 ms)Flushing 1588230740/info: creating writer at 1732758306716Flushing 1588230740/info: appending metadata at 1732758306744 (+28 ms)Flushing 1588230740/info: closing flushed file at 1732758306744Flushing 1588230740/ns: creating writer at 1732758306749 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732758306769 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732758306769Flushing 1588230740/table: creating writer at 1732758306774 (+5 ms)Flushing 1588230740/table: appending metadata at 1732758306794 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732758306794Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@110ccc: reopening flushed file at 1732758306798 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6787982e: reopening flushed file at 1732758306801 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f8f4962: reopening flushed file at 1732758306804 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=15, compaction requested=false at 1732758306808 (+4 ms)Writing region close event to WAL at 1732758306810 (+2 ms)Running coprocessor post-close hooks at 1732758306815 (+5 ms)Closed at 1732758306816 (+1 ms) 2024-11-28T01:45:06,816 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T01:45:06,914 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(976): stopping server 9df3f25f5867,35035,1732758301902; all regions closed. 2024-11-28T01:45:06,914 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(976): stopping server 9df3f25f5867,32859,1732758302059; all regions closed. 2024-11-28T01:45:06,920 DEBUG [RS:1;9df3f25f5867:32859 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/oldWALs 2024-11-28T01:45:06,920 DEBUG [RS:0;9df3f25f5867:35035 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/oldWALs 2024-11-28T01:45:06,920 INFO [RS:1;9df3f25f5867:32859 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9df3f25f5867%2C32859%2C1732758302059.meta:.meta(num 1732758303887) 2024-11-28T01:45:06,920 INFO [RS:0;9df3f25f5867:35035 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9df3f25f5867%2C35035%2C1732758301902:(num 1732758303542) 2024-11-28T01:45:06,920 DEBUG [RS:0;9df3f25f5867:35035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:06,920 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T01:45:06,920 INFO [RS:0;9df3f25f5867:35035 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T01:45:06,920 INFO [RS:0;9df3f25f5867:35035 {}] hbase.ChoreService(370): Chore service for: regionserver/9df3f25f5867:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-28T01:45:06,921 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T01:45:06,921 INFO [regionserver/9df3f25f5867:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T01:45:06,921 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T01:45:06,921 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T01:45:06,921 INFO [RS:0;9df3f25f5867:35035 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T01:45:06,922 INFO [RS:0;9df3f25f5867:35035 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35035 2024-11-28T01:45:06,922 DEBUG [RS:1;9df3f25f5867:32859 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/oldWALs 2024-11-28T01:45:06,922 INFO [RS:1;9df3f25f5867:32859 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9df3f25f5867%2C32859%2C1732758302059:(num 1732758303537) 2024-11-28T01:45:06,922 DEBUG [RS:1;9df3f25f5867:32859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:06,922 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T01:45:06,922 INFO [RS:1;9df3f25f5867:32859 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T01:45:06,922 INFO [RS:1;9df3f25f5867:32859 {}] hbase.ChoreService(370): Chore service for: regionserver/9df3f25f5867:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-28T01:45:06,922 INFO [RS:1;9df3f25f5867:32859 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T01:45:06,922 INFO [regionserver/9df3f25f5867:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T01:45:06,923 INFO [RS:1;9df3f25f5867:32859 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32859 2024-11-28T01:45:06,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9df3f25f5867,35035,1732758301902 2024-11-28T01:45:06,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T01:45:06,927 INFO [RS:0;9df3f25f5867:35035 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T01:45:06,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9df3f25f5867,32859,1732758302059 2024-11-28T01:45:06,927 INFO [RS:1;9df3f25f5867:32859 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T01:45:06,928 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9df3f25f5867,35035,1732758301902] 2024-11-28T01:45:06,930 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9df3f25f5867,35035,1732758301902 already deleted, retry=false 2024-11-28T01:45:06,931 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9df3f25f5867,35035,1732758301902 expired; onlineServers=1 2024-11-28T01:45:06,931 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9df3f25f5867,32859,1732758302059] 2024-11-28T01:45:06,932 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9df3f25f5867,32859,1732758302059 already deleted, retry=false 2024-11-28T01:45:06,932 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9df3f25f5867,32859,1732758302059 expired; onlineServers=0 2024-11-28T01:45:06,932 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9df3f25f5867,40245,1732758301060' ***** 2024-11-28T01:45:06,932 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-28T01:45:06,932 INFO [M:0;9df3f25f5867:40245 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T01:45:06,933 INFO [M:0;9df3f25f5867:40245 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T01:45:06,933 DEBUG [M:0;9df3f25f5867:40245 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-28T01:45:06,933 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-28T01:45:06,933 DEBUG [M:0;9df3f25f5867:40245 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-28T01:45:06,933 DEBUG [master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.large.0-1732758303166 {}] cleaner.HFileCleaner(306): Exit Thread[master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.large.0-1732758303166,5,FailOnTimeoutGroup] 2024-11-28T01:45:06,933 DEBUG [master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.small.0-1732758303167 {}] cleaner.HFileCleaner(306): Exit Thread[master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.small.0-1732758303167,5,FailOnTimeoutGroup] 2024-11-28T01:45:06,933 INFO [M:0;9df3f25f5867:40245 {}] hbase.ChoreService(370): Chore service for: master/9df3f25f5867:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-28T01:45:06,933 INFO [M:0;9df3f25f5867:40245 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T01:45:06,933 DEBUG [M:0;9df3f25f5867:40245 {}] master.HMaster(1795): Stopping service threads 2024-11-28T01:45:06,934 INFO [M:0;9df3f25f5867:40245 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-28T01:45:06,934 INFO [M:0;9df3f25f5867:40245 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T01:45:06,935 INFO [M:0;9df3f25f5867:40245 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T01:45:06,935 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-28T01:45:06,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-28T01:45:06,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:06,936 DEBUG [M:0;9df3f25f5867:40245 {}] zookeeper.ZKUtil(347): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-28T01:45:06,936 WARN [M:0;9df3f25f5867:40245 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-28T01:45:06,936 INFO [M:0;9df3f25f5867:40245 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/.lastflushedseqids 2024-11-28T01:45:06,946 INFO [M:0;9df3f25f5867:40245 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T01:45:06,946 INFO [M:0;9df3f25f5867:40245 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-28T01:45:06,946 DEBUG [M:0;9df3f25f5867:40245 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T01:45:06,946 INFO [M:0;9df3f25f5867:40245 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:45:06,946 DEBUG [M:0;9df3f25f5867:40245 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:45:06,946 DEBUG [M:0;9df3f25f5867:40245 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T01:45:06,946 DEBUG [M:0;9df3f25f5867:40245 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:45:06,947 INFO [M:0;9df3f25f5867:40245 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-28T01:45:06,957 DEBUG [M:0;9df3f25f5867:40245 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 79bd3659e64044479d5f0e5b8fe077c8 2024-11-28T01:45:06,971 DEBUG [M:0;9df3f25f5867:40245 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/79bd3659e64044479d5f0e5b8fe077c8 is 82, key is hbase:meta,,1/info:regioninfo/1732758303954/Put/seqid=0 2024-11-28T01:45:06,972 INFO [M:0;9df3f25f5867:40245 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/79bd3659e64044479d5f0e5b8fe077c8 2024-11-28T01:45:06,982 DEBUG [M:0;9df3f25f5867:40245 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on caacd988a3e5462c8bab2f6648742cd0 2024-11-28T01:45:06,996 DEBUG [M:0;9df3f25f5867:40245 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/caacd988a3e5462c8bab2f6648742cd0 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732758304741/Put/seqid=0 2024-11-28T01:45:06,997 INFO [M:0;9df3f25f5867:40245 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/caacd988a3e5462c8bab2f6648742cd0 2024-11-28T01:45:07,006 DEBUG [M:0;9df3f25f5867:40245 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 100cc0332dab4c45921357e64c63d484 2024-11-28T01:45:07,019 DEBUG [M:0;9df3f25f5867:40245 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/100cc0332dab4c45921357e64c63d484 is 69, key is 9df3f25f5867,32859,1732758302059/rs:state/1732758303297/Put/seqid=0 2024-11-28T01:45:07,020 INFO [M:0;9df3f25f5867:40245 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/100cc0332dab4c45921357e64c63d484 2024-11-28T01:45:07,022 DEBUG [M:0;9df3f25f5867:40245 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/79bd3659e64044479d5f0e5b8fe077c8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/79bd3659e64044479d5f0e5b8fe077c8 2024-11-28T01:45:07,023 INFO [M:0;9df3f25f5867:40245 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/79bd3659e64044479d5f0e5b8fe077c8, entries=8, sequenceid=84, filesize=5.5 K 2024-11-28T01:45:07,024 DEBUG [M:0;9df3f25f5867:40245 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/caacd988a3e5462c8bab2f6648742cd0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/caacd988a3e5462c8bab2f6648742cd0 2024-11-28T01:45:07,025 INFO [M:0;9df3f25f5867:40245 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/caacd988a3e5462c8bab2f6648742cd0, entries=9, sequenceid=84, filesize=6.5 K 2024-11-28T01:45:07,026 DEBUG [M:0;9df3f25f5867:40245 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/100cc0332dab4c45921357e64c63d484 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/100cc0332dab4c45921357e64c63d484 2024-11-28T01:45:07,028 INFO [M:0;9df3f25f5867:40245 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/100cc0332dab4c45921357e64c63d484, entries=2, sequenceid=84, filesize=5.1 K 2024-11-28T01:45:07,028 INFO [M:0;9df3f25f5867:40245 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=84, compaction requested=false 2024-11-28T01:45:07,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:45:07,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:45:07,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35035-0x1003bcd212c0001, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:45:07,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32859-0x1003bcd212c0002, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:45:07,030 INFO [RS:0;9df3f25f5867:35035 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T01:45:07,030 INFO [RS:1;9df3f25f5867:32859 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T01:45:07,031 INFO [RS:1;9df3f25f5867:32859 {}] regionserver.HRegionServer(1031): Exiting; stopping=9df3f25f5867,32859,1732758302059; zookeeper connection closed. 2024-11-28T01:45:07,031 INFO [RS:0;9df3f25f5867:35035 {}] regionserver.HRegionServer(1031): Exiting; stopping=9df3f25f5867,35035,1732758301902; zookeeper connection closed. 2024-11-28T01:45:07,031 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@36efc51c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@36efc51c 2024-11-28T01:45:07,031 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6ea1b767 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6ea1b767 2024-11-28T01:45:07,031 INFO [M:0;9df3f25f5867:40245 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:45:07,032 DEBUG [M:0;9df3f25f5867:40245 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732758306946Disabling compacts and flushes for region at 1732758306946Disabling writes for close at 1732758306946Obtaining lock to block concurrent updates at 1732758306947 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732758306947Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732758306947Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732758306947Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732758306948 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732758306971 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732758306971Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732758306974 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732758306995 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732758306995Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732758306999 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732758307018 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732758307018Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2906ac9: reopening flushed file at 1732758307021 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f916aaa: reopening flushed file at 1732758307024 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b7a638d: reopening flushed file at 1732758307026 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=84, compaction requested=false at 1732758307029 (+3 ms)Writing region close event to WAL at 1732758307031 (+2 ms)Closed at 1732758307031 2024-11-28T01:45:07,032 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-28T01:45:07,034 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T01:45:07,034 INFO [M:0;9df3f25f5867:40245 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-28T01:45:07,034 INFO [M:0;9df3f25f5867:40245 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40245 2024-11-28T01:45:07,034 INFO [M:0;9df3f25f5867:40245 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T01:45:07,137 INFO [M:0;9df3f25f5867:40245 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T01:45:07,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:45:07,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40245-0x1003bcd212c0000, quorum=127.0.0.1:54192, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:45:07,142 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-28T01:45:07,169 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-28T01:45:07,169 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:07,178 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/9df3f25f5867:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1732758305102 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732758306799 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/9df3f25f5867:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: hfile-prefetch-1732758306632 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/9df3f25f5867:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732758306803 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/9df3f25f5867:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=274 (was 289), ProcessCount=11 (was 11), AvailableMemoryMB=7523 (was 7918) 2024-11-28T01:45:07,185 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=274, ProcessCount=11, AvailableMemoryMB=7523 2024-11-28T01:45:07,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/hadoop-log-dir so I do NOT create it in target/test-data/174125f7-67da-32d4-925d-63271e41c80b 2024-11-28T01:45:07,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/hadoop-tmp-dir so I do NOT create it in target/test-data/174125f7-67da-32d4-925d-63271e41c80b 2024-11-28T01:45:07,186 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:07,189 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/cluster_e81a9123-313b-7c07-5f67-fbc05ea90c7f, deleteOnExit=true 2024-11-28T01:45:07,192 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/cluster_e81a9123-313b-7c07-5f67-fbc05ea90c7f/zookeeper_0, clientPort=55202, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/cluster_e81a9123-313b-7c07-5f67-fbc05ea90c7f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/cluster_e81a9123-313b-7c07-5f67-fbc05ea90c7f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-28T01:45:07,193 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55202 2024-11-28T01:45:07,207 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c with version=8 2024-11-28T01:45:07,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/hbase-staging 2024-11-28T01:45:07,210 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9df3f25f5867:0 server-side Connection retries=45 2024-11-28T01:45:07,210 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:07,210 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:07,210 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T01:45:07,210 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:07,210 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T01:45:07,210 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-28T01:45:07,210 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T01:45:07,211 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44337 2024-11-28T01:45:07,213 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44337 connecting to ZooKeeper ensemble=127.0.0.1:55202 2024-11-28T01:45:07,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:443370x0, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T01:45:07,219 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44337-0x1003bcd3a5e0000 connected 2024-11-28T01:45:07,234 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:07,234 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:07,235 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T01:45:07,235 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c, hbase.cluster.distributed=false 2024-11-28T01:45:07,238 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T01:45:07,238 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44337 2024-11-28T01:45:07,239 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44337 2024-11-28T01:45:07,239 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44337 2024-11-28T01:45:07,240 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44337 2024-11-28T01:45:07,240 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44337 2024-11-28T01:45:07,259 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9df3f25f5867:0 server-side Connection retries=45 2024-11-28T01:45:07,259 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:07,260 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:07,260 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T01:45:07,260 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:07,260 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T01:45:07,260 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T01:45:07,260 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T01:45:07,261 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37707 2024-11-28T01:45:07,263 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37707 connecting to ZooKeeper ensemble=127.0.0.1:55202 2024-11-28T01:45:07,263 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:07,263 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:07,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:377070x0, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T01:45:07,267 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37707-0x1003bcd3a5e0001 connected 2024-11-28T01:45:07,267 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T01:45:07,268 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T01:45:07,272 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/bucket.cache 2024-11-28T01:45:07,272 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-28T01:45:07,272 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-28T01:45:07,272 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-28T01:45:07,273 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-28T01:45:07,273 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-28T01:45:07,273 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-28T01:45:07,273 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-28T01:45:07,274 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/bucket.persistence is complete. 2024-11-28T01:45:07,274 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T01:45:07,275 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T01:45:07,276 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T01:45:07,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37707 2024-11-28T01:45:07,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37707 2024-11-28T01:45:07,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37707 2024-11-28T01:45:07,278 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37707 2024-11-28T01:45:07,278 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37707 2024-11-28T01:45:07,296 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9df3f25f5867:0 server-side Connection retries=45 2024-11-28T01:45:07,296 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:07,296 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:07,296 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T01:45:07,296 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:07,296 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T01:45:07,296 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T01:45:07,296 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T01:45:07,297 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45483 2024-11-28T01:45:07,299 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45483 connecting to ZooKeeper ensemble=127.0.0.1:55202 2024-11-28T01:45:07,299 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:07,299 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:07,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:454830x0, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T01:45:07,302 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45483-0x1003bcd3a5e0002 connected 2024-11-28T01:45:07,303 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T01:45:07,303 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T01:45:07,304 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/bucket.cache 2024-11-28T01:45:07,304 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-28T01:45:07,304 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-28T01:45:07,304 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-28T01:45:07,308 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-28T01:45:07,308 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-28T01:45:07,308 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-28T01:45:07,308 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/bucket.persistence is complete. 2024-11-28T01:45:07,309 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-28T01:45:07,312 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T01:45:07,313 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T01:45:07,314 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T01:45:07,320 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45483 2024-11-28T01:45:07,320 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45483 2024-11-28T01:45:07,320 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45483 2024-11-28T01:45:07,325 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45483 2024-11-28T01:45:07,325 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45483 2024-11-28T01:45:07,338 DEBUG [M:0;9df3f25f5867:44337 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9df3f25f5867:44337 2024-11-28T01:45:07,339 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9df3f25f5867,44337,1732758307209 2024-11-28T01:45:07,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T01:45:07,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T01:45:07,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T01:45:07,342 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9df3f25f5867,44337,1732758307209 2024-11-28T01:45:07,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T01:45:07,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T01:45:07,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:07,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:07,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:07,346 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T01:45:07,346 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9df3f25f5867,44337,1732758307209 from backup master directory 2024-11-28T01:45:07,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9df3f25f5867,44337,1732758307209 2024-11-28T01:45:07,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T01:45:07,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T01:45:07,348 WARN [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T01:45:07,348 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9df3f25f5867,44337,1732758307209 2024-11-28T01:45:07,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T01:45:07,350 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/hbase.id] with ID: 239d6f20-9868-46de-9e93-c1fd463a2e4a 2024-11-28T01:45:07,350 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/.tmp/hbase.id 2024-11-28T01:45:07,358 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/hbase.id] 2024-11-28T01:45:07,402 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:07,403 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-28T01:45:07,404 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-28T01:45:07,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:07,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:07,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:07,413 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T01:45:07,414 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-28T01:45:07,414 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T01:45:07,435 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store 2024-11-28T01:45:07,442 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:45:07,442 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T01:45:07,442 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:45:07,442 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:45:07,442 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T01:45:07,442 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:45:07,442 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:45:07,442 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732758307442Disabling compacts and flushes for region at 1732758307442Disabling writes for close at 1732758307442Writing region close event to WAL at 1732758307442Closed at 1732758307442 2024-11-28T01:45:07,442 WARN [master/9df3f25f5867:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/.initializing 2024-11-28T01:45:07,442 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/WALs/9df3f25f5867,44337,1732758307209 2024-11-28T01:45:07,448 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T01:45:07,451 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9df3f25f5867%2C44337%2C1732758307209, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/WALs/9df3f25f5867,44337,1732758307209, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/oldWALs, maxLogs=10 2024-11-28T01:45:07,477 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/WALs/9df3f25f5867,44337,1732758307209/9df3f25f5867%2C44337%2C1732758307209.1732758307452 2024-11-28T01:45:07,478 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T01:45:07,478 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:45:07,478 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:45:07,479 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:07,479 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:07,480 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:07,485 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-28T01:45:07,485 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:07,485 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:07,486 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:07,489 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-28T01:45:07,489 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:07,489 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:45:07,490 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:07,493 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-28T01:45:07,493 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:07,493 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:45:07,493 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:07,497 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-28T01:45:07,497 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:07,497 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:45:07,497 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:07,498 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:07,498 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:07,498 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:07,498 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:07,499 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T01:45:07,499 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T01:45:07,505 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T01:45:07,506 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61948101, jitterRate=-0.07690136134624481}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T01:45:07,507 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732758307479Initializing all the Stores at 1732758307479Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758307479Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758307480 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758307480Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758307480Cleaning up temporary data from old regions at 1732758307498 (+18 ms)Region opened successfully at 1732758307506 (+8 ms) 2024-11-28T01:45:07,507 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-28T01:45:07,511 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3599a22c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9df3f25f5867/172.17.0.2:0 2024-11-28T01:45:07,513 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-28T01:45:07,513 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-28T01:45:07,513 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-28T01:45:07,514 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T01:45:07,514 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T01:45:07,514 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-28T01:45:07,514 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-28T01:45:07,516 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-28T01:45:07,517 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-28T01:45:07,519 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-28T01:45:07,519 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-28T01:45:07,520 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-28T01:45:07,521 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-28T01:45:07,521 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-28T01:45:07,522 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-28T01:45:07,524 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-28T01:45:07,524 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-28T01:45:07,526 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-28T01:45:07,528 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-28T01:45:07,529 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-28T01:45:07,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T01:45:07,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T01:45:07,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T01:45:07,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:07,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:07,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:07,532 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9df3f25f5867,44337,1732758307209, sessionid=0x1003bcd3a5e0000, setting cluster-up flag (Was=false) 2024-11-28T01:45:07,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:07,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:07,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:07,540 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-28T01:45:07,541 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9df3f25f5867,44337,1732758307209 2024-11-28T01:45:07,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:07,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:07,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:07,549 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-28T01:45:07,550 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9df3f25f5867,44337,1732758307209 2024-11-28T01:45:07,551 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-28T01:45:07,553 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T01:45:07,554 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-28T01:45:07,555 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-28T01:45:07,555 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9df3f25f5867,44337,1732758307209 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-28T01:45:07,558 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9df3f25f5867:0, corePoolSize=5, maxPoolSize=5 2024-11-28T01:45:07,558 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9df3f25f5867:0, corePoolSize=5, maxPoolSize=5 2024-11-28T01:45:07,558 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9df3f25f5867:0, corePoolSize=5, maxPoolSize=5 2024-11-28T01:45:07,558 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9df3f25f5867:0, corePoolSize=5, maxPoolSize=5 2024-11-28T01:45:07,558 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9df3f25f5867:0, corePoolSize=10, maxPoolSize=10 2024-11-28T01:45:07,558 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,559 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9df3f25f5867:0, corePoolSize=2, maxPoolSize=2 2024-11-28T01:45:07,559 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,560 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T01:45:07,561 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-28T01:45:07,561 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:07,561 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-28T01:45:07,564 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732758337564 2024-11-28T01:45:07,565 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-28T01:45:07,565 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-28T01:45:07,565 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-28T01:45:07,565 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-28T01:45:07,565 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-28T01:45:07,565 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-28T01:45:07,565 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,566 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-28T01:45:07,566 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-28T01:45:07,566 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-28T01:45:07,566 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-28T01:45:07,566 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-28T01:45:07,567 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.large.0-1732758307566,5,FailOnTimeoutGroup] 2024-11-28T01:45:07,567 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.small.0-1732758307567,5,FailOnTimeoutGroup] 2024-11-28T01:45:07,567 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,567 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-28T01:45:07,567 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,567 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,584 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-28T01:45:07,584 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c 2024-11-28T01:45:07,596 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:45:07,596 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-28T01:45:07,601 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T01:45:07,601 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:07,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:07,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-28T01:45:07,605 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T01:45:07,606 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:07,606 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:07,606 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-28T01:45:07,609 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T01:45:07,609 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:07,609 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:07,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-28T01:45:07,613 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T01:45:07,613 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:07,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:07,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T01:45:07,614 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740 2024-11-28T01:45:07,614 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740 2024-11-28T01:45:07,614 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T01:45:07,614 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T01:45:07,614 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T01:45:07,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T01:45:07,621 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T01:45:07,621 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68663597, jitterRate=0.023167327046394348}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T01:45:07,621 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732758307596Initializing all the Stores at 1732758307596Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758307596Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758307596Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758307596Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758307596Cleaning up temporary data from old regions at 1732758307614 (+18 ms)Region opened successfully at 1732758307621 (+7 ms) 2024-11-28T01:45:07,622 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T01:45:07,622 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T01:45:07,622 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T01:45:07,622 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T01:45:07,622 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T01:45:07,622 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T01:45:07,622 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732758307622Disabling compacts and flushes for region at 1732758307622Disabling writes for close at 1732758307622Writing region close event to WAL at 1732758307622Closed at 1732758307622 2024-11-28T01:45:07,623 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T01:45:07,623 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-28T01:45:07,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T01:45:07,624 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T01:45:07,625 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-28T01:45:07,627 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.HRegionServer(746): ClusterId : 239d6f20-9868-46de-9e93-c1fd463a2e4a 2024-11-28T01:45:07,628 DEBUG [RS:1;9df3f25f5867:45483 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T01:45:07,628 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer(746): ClusterId : 239d6f20-9868-46de-9e93-c1fd463a2e4a 2024-11-28T01:45:07,628 DEBUG [RS:0;9df3f25f5867:37707 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T01:45:07,630 DEBUG [RS:1;9df3f25f5867:45483 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T01:45:07,631 DEBUG [RS:1;9df3f25f5867:45483 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T01:45:07,631 DEBUG [RS:0;9df3f25f5867:37707 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T01:45:07,631 DEBUG [RS:0;9df3f25f5867:37707 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T01:45:07,633 DEBUG [RS:1;9df3f25f5867:45483 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T01:45:07,634 DEBUG [RS:1;9df3f25f5867:45483 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd3b69f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9df3f25f5867/172.17.0.2:0 2024-11-28T01:45:07,634 DEBUG [RS:0;9df3f25f5867:37707 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T01:45:07,634 DEBUG [RS:0;9df3f25f5867:37707 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cda9c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9df3f25f5867/172.17.0.2:0 2024-11-28T01:45:07,647 DEBUG [RS:1;9df3f25f5867:45483 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9df3f25f5867:45483 2024-11-28T01:45:07,647 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T01:45:07,647 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T01:45:07,647 DEBUG [RS:1;9df3f25f5867:45483 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T01:45:07,647 DEBUG [RS:0;9df3f25f5867:37707 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9df3f25f5867:37707 2024-11-28T01:45:07,647 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T01:45:07,647 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T01:45:07,647 DEBUG [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T01:45:07,648 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.HRegionServer(2659): reportForDuty to master=9df3f25f5867,44337,1732758307209 with port=45483, startcode=1732758307295 2024-11-28T01:45:07,648 DEBUG [RS:1;9df3f25f5867:45483 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T01:45:07,648 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer(2659): reportForDuty to master=9df3f25f5867,44337,1732758307209 with port=37707, startcode=1732758307259 2024-11-28T01:45:07,648 DEBUG [RS:0;9df3f25f5867:37707 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T01:45:07,653 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40879, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T01:45:07,653 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60949, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T01:45:07,653 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44337 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9df3f25f5867,37707,1732758307259 2024-11-28T01:45:07,653 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44337 {}] master.ServerManager(517): Registering regionserver=9df3f25f5867,37707,1732758307259 2024-11-28T01:45:07,655 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44337 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9df3f25f5867,45483,1732758307295 2024-11-28T01:45:07,655 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44337 {}] master.ServerManager(517): Registering regionserver=9df3f25f5867,45483,1732758307295 2024-11-28T01:45:07,655 DEBUG [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c 2024-11-28T01:45:07,655 DEBUG [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-28T01:45:07,655 DEBUG [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T01:45:07,657 DEBUG [RS:1;9df3f25f5867:45483 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c 2024-11-28T01:45:07,657 DEBUG [RS:1;9df3f25f5867:45483 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-28T01:45:07,657 DEBUG [RS:1;9df3f25f5867:45483 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T01:45:07,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T01:45:07,658 DEBUG [RS:0;9df3f25f5867:37707 {}] zookeeper.ZKUtil(111): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9df3f25f5867,37707,1732758307259 2024-11-28T01:45:07,658 WARN [RS:0;9df3f25f5867:37707 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T01:45:07,658 INFO [RS:0;9df3f25f5867:37707 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T01:45:07,658 DEBUG [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,37707,1732758307259 2024-11-28T01:45:07,662 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9df3f25f5867,37707,1732758307259] 2024-11-28T01:45:07,662 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9df3f25f5867,45483,1732758307295] 2024-11-28T01:45:07,662 DEBUG [RS:1;9df3f25f5867:45483 {}] zookeeper.ZKUtil(111): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9df3f25f5867,45483,1732758307295 2024-11-28T01:45:07,662 WARN [RS:1;9df3f25f5867:45483 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T01:45:07,663 INFO [RS:1;9df3f25f5867:45483 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T01:45:07,663 DEBUG [RS:1;9df3f25f5867:45483 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,45483,1732758307295 2024-11-28T01:45:07,667 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T01:45:07,669 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T01:45:07,670 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T01:45:07,671 INFO [RS:0;9df3f25f5867:37707 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T01:45:07,671 INFO [RS:0;9df3f25f5867:37707 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,671 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T01:45:07,671 INFO [RS:0;9df3f25f5867:37707 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T01:45:07,672 INFO [RS:0;9df3f25f5867:37707 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,672 DEBUG [RS:0;9df3f25f5867:37707 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,672 DEBUG [RS:0;9df3f25f5867:37707 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,672 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T01:45:07,672 DEBUG [RS:0;9df3f25f5867:37707 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,672 DEBUG [RS:0;9df3f25f5867:37707 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,672 DEBUG [RS:0;9df3f25f5867:37707 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,672 DEBUG [RS:0;9df3f25f5867:37707 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9df3f25f5867:0, corePoolSize=2, maxPoolSize=2 2024-11-28T01:45:07,672 DEBUG [RS:0;9df3f25f5867:37707 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,672 INFO [RS:1;9df3f25f5867:45483 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T01:45:07,672 DEBUG [RS:0;9df3f25f5867:37707 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,673 INFO [RS:1;9df3f25f5867:45483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,673 DEBUG [RS:0;9df3f25f5867:37707 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,673 DEBUG [RS:0;9df3f25f5867:37707 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,673 DEBUG [RS:0;9df3f25f5867:37707 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,673 DEBUG [RS:0;9df3f25f5867:37707 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,673 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T01:45:07,673 DEBUG [RS:0;9df3f25f5867:37707 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9df3f25f5867:0, corePoolSize=3, maxPoolSize=3 2024-11-28T01:45:07,673 DEBUG [RS:0;9df3f25f5867:37707 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0, corePoolSize=3, maxPoolSize=3 2024-11-28T01:45:07,673 INFO [RS:1;9df3f25f5867:45483 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T01:45:07,674 INFO [RS:1;9df3f25f5867:45483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,674 DEBUG [RS:1;9df3f25f5867:45483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,674 INFO [RS:0;9df3f25f5867:37707 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,674 DEBUG [RS:1;9df3f25f5867:45483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,674 INFO [RS:0;9df3f25f5867:37707 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,674 DEBUG [RS:1;9df3f25f5867:45483 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,674 INFO [RS:0;9df3f25f5867:37707 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,674 DEBUG [RS:1;9df3f25f5867:45483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,674 INFO [RS:0;9df3f25f5867:37707 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,674 DEBUG [RS:1;9df3f25f5867:45483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,674 INFO [RS:0;9df3f25f5867:37707 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,674 DEBUG [RS:1;9df3f25f5867:45483 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9df3f25f5867:0, corePoolSize=2, maxPoolSize=2 2024-11-28T01:45:07,674 INFO [RS:0;9df3f25f5867:37707 {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,37707,1732758307259-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T01:45:07,674 DEBUG [RS:1;9df3f25f5867:45483 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,675 DEBUG [RS:1;9df3f25f5867:45483 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,675 DEBUG [RS:1;9df3f25f5867:45483 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,675 DEBUG [RS:1;9df3f25f5867:45483 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,675 DEBUG [RS:1;9df3f25f5867:45483 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,675 DEBUG [RS:1;9df3f25f5867:45483 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:07,675 DEBUG [RS:1;9df3f25f5867:45483 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9df3f25f5867:0, corePoolSize=3, maxPoolSize=3 2024-11-28T01:45:07,675 DEBUG [RS:1;9df3f25f5867:45483 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0, corePoolSize=3, maxPoolSize=3 2024-11-28T01:45:07,676 INFO [RS:1;9df3f25f5867:45483 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,676 INFO [RS:1;9df3f25f5867:45483 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,676 INFO [RS:1;9df3f25f5867:45483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,676 INFO [RS:1;9df3f25f5867:45483 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,676 INFO [RS:1;9df3f25f5867:45483 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,676 INFO [RS:1;9df3f25f5867:45483 {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,45483,1732758307295-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T01:45:07,700 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T01:45:07,700 INFO [RS:0;9df3f25f5867:37707 {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,37707,1732758307259-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,700 INFO [RS:0;9df3f25f5867:37707 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,700 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.Replication(171): 9df3f25f5867,37707,1732758307259 started 2024-11-28T01:45:07,701 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T01:45:07,701 INFO [RS:1;9df3f25f5867:45483 {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,45483,1732758307295-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,701 INFO [RS:1;9df3f25f5867:45483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,702 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.Replication(171): 9df3f25f5867,45483,1732758307295 started 2024-11-28T01:45:07,716 INFO [RS:0;9df3f25f5867:37707 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,717 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer(1482): Serving as 9df3f25f5867,37707,1732758307259, RpcServer on 9df3f25f5867/172.17.0.2:37707, sessionid=0x1003bcd3a5e0001 2024-11-28T01:45:07,717 DEBUG [RS:0;9df3f25f5867:37707 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T01:45:07,717 DEBUG [RS:0;9df3f25f5867:37707 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9df3f25f5867,37707,1732758307259 2024-11-28T01:45:07,717 DEBUG [RS:0;9df3f25f5867:37707 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9df3f25f5867,37707,1732758307259' 2024-11-28T01:45:07,717 DEBUG [RS:0;9df3f25f5867:37707 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T01:45:07,717 INFO [RS:1;9df3f25f5867:45483 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:07,718 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.HRegionServer(1482): Serving as 9df3f25f5867,45483,1732758307295, RpcServer on 9df3f25f5867/172.17.0.2:45483, sessionid=0x1003bcd3a5e0002 2024-11-28T01:45:07,718 DEBUG [RS:0;9df3f25f5867:37707 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T01:45:07,718 DEBUG [RS:1;9df3f25f5867:45483 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T01:45:07,718 DEBUG [RS:1;9df3f25f5867:45483 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9df3f25f5867,45483,1732758307295 2024-11-28T01:45:07,718 DEBUG [RS:1;9df3f25f5867:45483 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9df3f25f5867,45483,1732758307295' 2024-11-28T01:45:07,718 DEBUG [RS:1;9df3f25f5867:45483 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T01:45:07,718 DEBUG [RS:0;9df3f25f5867:37707 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T01:45:07,719 DEBUG [RS:0;9df3f25f5867:37707 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T01:45:07,719 DEBUG [RS:0;9df3f25f5867:37707 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9df3f25f5867,37707,1732758307259 2024-11-28T01:45:07,719 DEBUG [RS:0;9df3f25f5867:37707 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9df3f25f5867,37707,1732758307259' 2024-11-28T01:45:07,719 DEBUG [RS:0;9df3f25f5867:37707 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T01:45:07,719 DEBUG [RS:1;9df3f25f5867:45483 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T01:45:07,719 DEBUG [RS:0;9df3f25f5867:37707 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T01:45:07,719 DEBUG [RS:1;9df3f25f5867:45483 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T01:45:07,719 DEBUG [RS:1;9df3f25f5867:45483 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T01:45:07,719 DEBUG [RS:1;9df3f25f5867:45483 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9df3f25f5867,45483,1732758307295 2024-11-28T01:45:07,719 DEBUG [RS:1;9df3f25f5867:45483 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9df3f25f5867,45483,1732758307295' 2024-11-28T01:45:07,719 DEBUG [RS:1;9df3f25f5867:45483 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T01:45:07,720 DEBUG [RS:0;9df3f25f5867:37707 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T01:45:07,720 INFO [RS:0;9df3f25f5867:37707 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T01:45:07,720 INFO [RS:0;9df3f25f5867:37707 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T01:45:07,720 DEBUG [RS:1;9df3f25f5867:45483 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T01:45:07,721 DEBUG [RS:1;9df3f25f5867:45483 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T01:45:07,721 INFO [RS:1;9df3f25f5867:45483 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T01:45:07,721 INFO [RS:1;9df3f25f5867:45483 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T01:45:07,775 WARN [9df3f25f5867:44337 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T01:45:07,821 INFO [RS:0;9df3f25f5867:37707 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T01:45:07,822 INFO [RS:1;9df3f25f5867:45483 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T01:45:07,822 INFO [RS:0;9df3f25f5867:37707 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9df3f25f5867%2C37707%2C1732758307259, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,37707,1732758307259, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/oldWALs, maxLogs=55 2024-11-28T01:45:07,823 INFO [RS:1;9df3f25f5867:45483 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9df3f25f5867%2C45483%2C1732758307295, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,45483,1732758307295, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/oldWALs, maxLogs=55 2024-11-28T01:45:07,844 INFO [RS:1;9df3f25f5867:45483 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,45483,1732758307295/9df3f25f5867%2C45483%2C1732758307295.1732758307825 2024-11-28T01:45:07,844 DEBUG [RS:1;9df3f25f5867:45483 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T01:45:07,850 INFO [RS:0;9df3f25f5867:37707 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,37707,1732758307259/9df3f25f5867%2C37707%2C1732758307259.1732758307825 2024-11-28T01:45:07,850 DEBUG [RS:0;9df3f25f5867:37707 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T01:45:08,026 DEBUG [9df3f25f5867:44337 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-28T01:45:08,026 DEBUG [9df3f25f5867:44337 {}] balancer.BalancerClusterState(204): Hosts are {9df3f25f5867=0} racks are {/default-rack=0} 2024-11-28T01:45:08,027 DEBUG [9df3f25f5867:44337 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T01:45:08,027 DEBUG [9df3f25f5867:44337 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T01:45:08,027 DEBUG [9df3f25f5867:44337 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T01:45:08,027 DEBUG [9df3f25f5867:44337 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T01:45:08,027 INFO [9df3f25f5867:44337 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T01:45:08,027 INFO [9df3f25f5867:44337 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T01:45:08,027 DEBUG [9df3f25f5867:44337 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T01:45:08,028 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9df3f25f5867,37707,1732758307259 2024-11-28T01:45:08,029 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9df3f25f5867,37707,1732758307259, state=OPENING 2024-11-28T01:45:08,031 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-28T01:45:08,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:08,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:08,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:08,033 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T01:45:08,033 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:08,033 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:08,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9df3f25f5867,37707,1732758307259}] 2024-11-28T01:45:08,033 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:08,187 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T01:45:08,189 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46155, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T01:45:08,191 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T01:45:08,192 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T01:45:08,192 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T01:45:08,194 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9df3f25f5867%2C37707%2C1732758307259.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,37707,1732758307259, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/oldWALs, maxLogs=55 2024-11-28T01:45:08,222 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,37707,1732758307259/9df3f25f5867%2C37707%2C1732758307259.meta.1732758308195.meta 2024-11-28T01:45:08,223 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T01:45:08,223 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:45:08,224 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T01:45:08,224 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T01:45:08,224 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T01:45:08,224 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T01:45:08,224 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:45:08,224 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T01:45:08,224 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T01:45:08,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-28T01:45:08,226 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T01:45:08,226 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:08,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:08,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-28T01:45:08,227 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T01:45:08,227 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:08,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:08,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-28T01:45:08,228 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T01:45:08,228 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:08,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:08,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-28T01:45:08,229 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T01:45:08,229 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:08,229 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:08,229 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T01:45:08,230 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740 2024-11-28T01:45:08,230 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740 2024-11-28T01:45:08,230 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T01:45:08,230 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T01:45:08,230 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T01:45:08,231 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T01:45:08,232 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65863363, jitterRate=-0.01855941116809845}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T01:45:08,232 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T01:45:08,233 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732758308225Writing region info on filesystem at 1732758308225Initializing all the Stores at 1732758308225Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758308225Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758308225Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758308225Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758308225Cleaning up temporary data from old regions at 1732758308230 (+5 ms)Running coprocessor post-open hooks at 1732758308232 (+2 ms)Region opened successfully at 1732758308233 (+1 ms) 2024-11-28T01:45:08,233 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732758308186 2024-11-28T01:45:08,236 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T01:45:08,236 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T01:45:08,237 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9df3f25f5867,37707,1732758307259 2024-11-28T01:45:08,238 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9df3f25f5867,37707,1732758307259, state=OPEN 2024-11-28T01:45:08,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T01:45:08,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T01:45:08,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T01:45:08,240 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:08,240 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:08,240 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:08,240 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9df3f25f5867,37707,1732758307259 2024-11-28T01:45:08,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-28T01:45:08,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9df3f25f5867,37707,1732758307259 in 207 msec 2024-11-28T01:45:08,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-28T01:45:08,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 620 msec 2024-11-28T01:45:08,245 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T01:45:08,245 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-28T01:45:08,249 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:45:08,249 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,37707,1732758307259, seqNum=-1] 2024-11-28T01:45:08,250 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:45:08,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53147, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:45:08,258 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 704 msec 2024-11-28T01:45:08,258 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732758308258, completionTime=-1 2024-11-28T01:45:08,258 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-28T01:45:08,258 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T01:45:08,261 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T01:45:08,261 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732758368261 2024-11-28T01:45:08,261 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732758428261 2024-11-28T01:45:08,261 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-28T01:45:08,262 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-28T01:45:08,262 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,44337,1732758307209-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:08,262 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,44337,1732758307209-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:08,262 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,44337,1732758307209-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:08,263 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9df3f25f5867:44337, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:08,263 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:08,263 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:08,266 DEBUG [master/9df3f25f5867:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-28T01:45:08,269 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.920sec 2024-11-28T01:45:08,269 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-28T01:45:08,269 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-28T01:45:08,269 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-28T01:45:08,269 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-28T01:45:08,269 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-28T01:45:08,269 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,44337,1732758307209-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T01:45:08,269 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,44337,1732758307209-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-28T01:45:08,270 DEBUG [master/9df3f25f5867:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-28T01:45:08,270 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-28T01:45:08,270 INFO [master/9df3f25f5867:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,44337,1732758307209-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:08,327 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:45:08,328 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9df3f25f5867,44337,-1 for getting cluster id 2024-11-28T01:45:08,328 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T01:45:08,330 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '239d6f20-9868-46de-9e93-c1fd463a2e4a' 2024-11-28T01:45:08,330 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T01:45:08,331 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "239d6f20-9868-46de-9e93-c1fd463a2e4a" 2024-11-28T01:45:08,331 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a0d9855, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:45:08,331 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9df3f25f5867,44337,-1] 2024-11-28T01:45:08,331 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T01:45:08,332 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:08,333 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T01:45:08,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34126543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T01:45:08,335 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:45:08,336 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,37707,1732758307259, seqNum=-1] 2024-11-28T01:45:08,337 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:45:08,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:45:08,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9df3f25f5867,44337,1732758307209 2024-11-28T01:45:08,341 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T01:45:08,343 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9df3f25f5867,44337,1732758307209 2024-11-28T01:45:08,343 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@48d86669 2024-11-28T01:45:08,343 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T01:45:08,345 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T01:45:08,345 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44337 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732758308341', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T01:45:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44337 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732758308341 2024-11-28T01:45:08,348 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732758308341 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T01:45:08,348 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:08,348 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44337 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732758308341" procId is: 4 2024-11-28T01:45:08,349 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732758308341 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T01:45:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44337 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T01:45:08,360 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732758308341-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 04c80cae9a2e31b89a0a10577aea4215, NAME => 'testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732758308341', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c 2024-11-28T01:45:08,367 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732758308341-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:45:08,368 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732758308341-pool-0 {}] regionserver.HRegion(1722): Closing 04c80cae9a2e31b89a0a10577aea4215, disabling compactions & flushes 2024-11-28T01:45:08,368 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732758308341-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:08,368 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732758308341-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:08,368 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732758308341-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. after waiting 0 ms 2024-11-28T01:45:08,368 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732758308341-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:08,368 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732758308341-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:08,368 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732758308341-pool-0 {}] regionserver.HRegion(1676): Region close journal for 04c80cae9a2e31b89a0a10577aea4215: Waiting for close lock at 1732758308368Disabling compacts and flushes for region at 1732758308368Disabling writes for close at 1732758308368Writing region close event to WAL at 1732758308368Closed at 1732758308368 2024-11-28T01:45:08,370 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732758308341 execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T01:45:08,370 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732758308370"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732758308370"}]},"ts":"1732758308370"} 2024-11-28T01:45:08,373 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T01:45:08,374 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732758308341 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T01:45:08,374 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732758308341","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758308374"}]},"ts":"1732758308374"} 2024-11-28T01:45:08,376 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732758308341, state=ENABLING in hbase:meta 2024-11-28T01:45:08,376 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {9df3f25f5867=0} racks are {/default-rack=0} 2024-11-28T01:45:08,377 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T01:45:08,377 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T01:45:08,377 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T01:45:08,377 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T01:45:08,377 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T01:45:08,377 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T01:45:08,377 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T01:45:08,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732758308341, region=04c80cae9a2e31b89a0a10577aea4215, ASSIGN}] 2024-11-28T01:45:08,378 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732758308341, region=04c80cae9a2e31b89a0a10577aea4215, ASSIGN 2024-11-28T01:45:08,379 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732758308341, region=04c80cae9a2e31b89a0a10577aea4215, ASSIGN; state=OFFLINE, location=9df3f25f5867,37707,1732758307259; forceNewPlan=false, retain=false 2024-11-28T01:45:08,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44337 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T01:45:08,530 INFO [9df3f25f5867:44337 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-28T01:45:08,530 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=04c80cae9a2e31b89a0a10577aea4215, regionState=OPENING, regionLocation=9df3f25f5867,37707,1732758307259 2024-11-28T01:45:08,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732758308341, region=04c80cae9a2e31b89a0a10577aea4215, ASSIGN because future has completed 2024-11-28T01:45:08,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04c80cae9a2e31b89a0a10577aea4215, server=9df3f25f5867,37707,1732758307259}] 2024-11-28T01:45:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44337 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T01:45:08,689 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:08,689 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 04c80cae9a2e31b89a0a10577aea4215, NAME => 'testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:45:08,690 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732758308341 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:08,690 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:45:08,690 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:08,690 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:08,691 INFO [StoreOpener-04c80cae9a2e31b89a0a10577aea4215-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:08,695 INFO [StoreOpener-04c80cae9a2e31b89a0a10577aea4215-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04c80cae9a2e31b89a0a10577aea4215 columnFamilyName family 2024-11-28T01:45:08,695 DEBUG [StoreOpener-04c80cae9a2e31b89a0a10577aea4215-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:08,695 INFO [StoreOpener-04c80cae9a2e31b89a0a10577aea4215-1 {}] regionserver.HStore(327): Store=04c80cae9a2e31b89a0a10577aea4215/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:45:08,695 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:08,696 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:08,696 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:08,697 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:08,697 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:08,698 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:08,704 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T01:45:08,705 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 04c80cae9a2e31b89a0a10577aea4215; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65784332, jitterRate=-0.019737064838409424}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T01:45:08,705 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:08,706 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 04c80cae9a2e31b89a0a10577aea4215: Running coprocessor pre-open hook at 1732758308690Writing region info on filesystem at 1732758308690Initializing all the Stores at 1732758308690Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758308690Cleaning up temporary data from old regions at 1732758308697 (+7 ms)Running coprocessor post-open hooks at 1732758308705 (+8 ms)Region opened successfully at 1732758308706 (+1 ms) 2024-11-28T01:45:08,706 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215., pid=6, masterSystemTime=1732758308686 2024-11-28T01:45:08,710 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:08,710 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:08,710 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=04c80cae9a2e31b89a0a10577aea4215, regionState=OPEN, openSeqNum=2, regionLocation=9df3f25f5867,37707,1732758307259 2024-11-28T01:45:08,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04c80cae9a2e31b89a0a10577aea4215, server=9df3f25f5867,37707,1732758307259 because future has completed 2024-11-28T01:45:08,716 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-28T01:45:08,716 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 04c80cae9a2e31b89a0a10577aea4215, server=9df3f25f5867,37707,1732758307259 in 180 msec 2024-11-28T01:45:08,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-28T01:45:08,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732758308341, region=04c80cae9a2e31b89a0a10577aea4215, ASSIGN in 339 msec 2024-11-28T01:45:08,718 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732758308341 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T01:45:08,718 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732758308341","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732758308718"}]},"ts":"1732758308718"} 2024-11-28T01:45:08,721 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732758308341, state=ENABLED in hbase:meta 2024-11-28T01:45:08,722 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732758308341 execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T01:45:08,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732758308341 in 376 msec 2024-11-28T01:45:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44337 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T01:45:08,979 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732758308341 completed 2024-11-28T01:45:08,979 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732758308341 get assigned. Timeout = 60000ms 2024-11-28T01:45:08,979 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T01:45:08,983 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732758308341 assigned to meta. Checking AM states. 2024-11-28T01:45:08,984 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T01:45:08,984 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732758308341 assigned. 2024-11-28T01:45:08,987 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732758308341', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215., hostname=9df3f25f5867,37707,1732758307259, seqNum=2] 2024-11-28T01:45:08,992 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 04c80cae9a2e31b89a0a10577aea4215 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-28T01:45:09,003 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 93048ea5a43a4ce795eca170b65d7f1e 2024-11-28T01:45:09,017 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/.tmp/family/93048ea5a43a4ce795eca170b65d7f1e is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-28T01:45:09,018 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/.tmp/family/93048ea5a43a4ce795eca170b65d7f1e 2024-11-28T01:45:09,021 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/.tmp/family/93048ea5a43a4ce795eca170b65d7f1e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/family/93048ea5a43a4ce795eca170b65d7f1e 2024-11-28T01:45:09,022 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/family/93048ea5a43a4ce795eca170b65d7f1e, delay=971 ms 2024-11-28T01:45:09,024 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/family/93048ea5a43a4ce795eca170b65d7f1e, entries=2, sequenceid=6, filesize=4.9 K 2024-11-28T01:45:09,024 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 04c80cae9a2e31b89a0a10577aea4215 in 32ms, sequenceid=6, compaction requested=false 2024-11-28T01:45:09,024 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732758308341' 2024-11-28T01:45:09,025 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 04c80cae9a2e31b89a0a10577aea4215: 2024-11-28T01:45:09,425 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T01:45:09,995 DEBUG [hfile-prefetch-1732758306632 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 93048ea5a43a4ce795eca170b65d7f1e_0. 2024-11-28T01:45:09,996 DEBUG [hfile-prefetch-1732758306632 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 93048ea5a43a4ce795eca170b65d7f1e_117. 2024-11-28T01:45:09,996 DEBUG [hfile-prefetch-1732758306632 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/family/93048ea5a43a4ce795eca170b65d7f1e, with total blocks 2, and data blocks 1 2024-11-28T01:45:09,996 DEBUG [hfile-prefetch-1732758306632 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-28T01:45:09,996 DEBUG [hfile-prefetch-1732758306632 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/family/93048ea5a43a4ce795eca170b65d7f1e, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-28T01:45:09,996 DEBUG [hfile-prefetch-1732758306632 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/family/93048ea5a43a4ce795eca170b65d7f1e. Will sleep 100ms and try the verification again. 2024-11-28T01:45:10,007 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-28T01:45:10,007 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-28T01:45:10,007 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T01:45:10,007 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T01:45:10,010 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-28T01:45:10,010 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732758310007 2024-11-28T01:45:10,010 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-28T01:45:10,025 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;9df3f25f5867:37707,5,FailOnTimeoutGroup] 2024-11-28T01:45:10,026 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9df3f25f5867,37707,1732758307259' ***** 2024-11-28T01:45:10,026 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-28T01:45:10,026 INFO [RS:0;9df3f25f5867:37707 {}] bucket.BucketCache(1738): Disabling cache 2024-11-28T01:45:10,026 INFO [RS:0;9df3f25f5867:37707 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/bucket.persistence 2024-11-28T01:45:10,026 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-28T01:45:10,026 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-28T01:45:10,026 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-28T01:45:10,026 INFO [RS:0;9df3f25f5867:37707 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-28T01:45:10,027 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-28T01:45:10,037 DEBUG [RS:0;9df3f25f5867:37707 {}] bucket.BucketCache(1487): Thread RS:0;9df3f25f5867:37707 started persisting bucket cache to file 2024-11-28T01:45:10,037 DEBUG [RS:0;9df3f25f5867:37707 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T01:45:10,037 DEBUG [RS:0;9df3f25f5867:37707 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T01:45:10,044 DEBUG [RS:0;9df3f25f5867:37707 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-28T01:45:10,044 DEBUG [RS:0;9df3f25f5867:37707 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732758310037 2024-11-28T01:45:10,044 DEBUG [RS:0;9df3f25f5867:37707 {}] bucket.BucketCache(1509): Thread RS:0;9df3f25f5867:37707 finished persisting bucket cache to file, renaming 2024-11-28T01:45:10,044 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T01:45:10,044 INFO [RS:0;9df3f25f5867:37707 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T01:45:10,044 INFO [RS:0;9df3f25f5867:37707 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T01:45:10,044 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T01:45:10,044 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer(3091): Received CLOSE for 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:10,045 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer(959): stopping server 9df3f25f5867,37707,1732758307259 2024-11-28T01:45:10,045 INFO [RS:0;9df3f25f5867:37707 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T01:45:10,045 INFO [RS:0;9df3f25f5867:37707 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9df3f25f5867:37707. 2024-11-28T01:45:10,045 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T01:45:10,045 DEBUG [RS:0;9df3f25f5867:37707 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:45:10,045 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 04c80cae9a2e31b89a0a10577aea4215, disabling compactions & flushes 2024-11-28T01:45:10,045 DEBUG [RS:0;9df3f25f5867:37707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:10,045 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:10,045 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:10,045 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T01:45:10,045 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. after waiting 0 ms 2024-11-28T01:45:10,045 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T01:45:10,045 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T01:45:10,045 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:10,045 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T01:45:10,046 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-28T01:45:10,046 DEBUG [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer(1325): Online Regions={04c80cae9a2e31b89a0a10577aea4215=testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215., 1588230740=hbase:meta,,1.1588230740} 2024-11-28T01:45:10,046 DEBUG [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer(1351): Waiting on 04c80cae9a2e31b89a0a10577aea4215, 1588230740 2024-11-28T01:45:10,046 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T01:45:10,046 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T01:45:10,046 DEBUG [StoreFileCloser-04c80cae9a2e31b89a0a10577aea4215-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/family/93048ea5a43a4ce795eca170b65d7f1e 2024-11-28T01:45:10,046 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T01:45:10,046 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T01:45:10,046 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T01:45:10,046 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T01:45:10,046 WARN [hfile-prefetch-1732758306632 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/family/93048ea5a43a4ce795eca170b65d7f1e, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-28T01:45:10,047 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-28T01:45:10,047 DEBUG [hfile-prefetch-1732758306632 {}] hfile.PrefetchExecutor(121): Prefetch completed for 93048ea5a43a4ce795eca170b65d7f1e 2024-11-28T01:45:10,051 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-28T01:45:10,051 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:10,052 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 04c80cae9a2e31b89a0a10577aea4215: Waiting for close lock at 1732758310045Running coprocessor pre-close hooks at 1732758310045Disabling compacts and flushes for region at 1732758310045Disabling writes for close at 1732758310045Writing region close event to WAL at 1732758310046 (+1 ms)Running coprocessor post-close hooks at 1732758310051 (+5 ms)Closed at 1732758310051 2024-11-28T01:45:10,052 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:10,057 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c43f7e06ef6d46959cd48fcbd49ee361 2024-11-28T01:45:10,071 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/info/c43f7e06ef6d46959cd48fcbd49ee361 is 209, key is testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215./info:regioninfo/1732758308710/Put/seqid=0 2024-11-28T01:45:10,072 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/info/c43f7e06ef6d46959cd48fcbd49ee361 2024-11-28T01:45:10,077 INFO [regionserver/9df3f25f5867:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T01:45:10,081 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 645334fdce0c40b79695102805830ff7 2024-11-28T01:45:10,101 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/ns/645334fdce0c40b79695102805830ff7 is 43, key is default/ns:d/1732758308252/Put/seqid=0 2024-11-28T01:45:10,102 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/ns/645334fdce0c40b79695102805830ff7 2024-11-28T01:45:10,111 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 74125ef68b814475bd714f5ca80f8bee 2024-11-28T01:45:10,128 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/table/74125ef68b814475bd714f5ca80f8bee is 80, key is testBlockEvictionOnGracefulStop1732758308341/table:state/1732758308718/Put/seqid=0 2024-11-28T01:45:10,128 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/table/74125ef68b814475bd714f5ca80f8bee 2024-11-28T01:45:10,131 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/info/c43f7e06ef6d46959cd48fcbd49ee361 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/info/c43f7e06ef6d46959cd48fcbd49ee361 2024-11-28T01:45:10,133 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/info/c43f7e06ef6d46959cd48fcbd49ee361, delay=950 ms 2024-11-28T01:45:10,134 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/info/c43f7e06ef6d46959cd48fcbd49ee361, entries=10, sequenceid=11, filesize=7.0 K 2024-11-28T01:45:10,134 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/ns/645334fdce0c40b79695102805830ff7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/ns/645334fdce0c40b79695102805830ff7 2024-11-28T01:45:10,135 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/ns/645334fdce0c40b79695102805830ff7, delay=943 ms 2024-11-28T01:45:10,136 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/ns/645334fdce0c40b79695102805830ff7, entries=2, sequenceid=11, filesize=5.0 K 2024-11-28T01:45:10,137 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/table/74125ef68b814475bd714f5ca80f8bee as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/table/74125ef68b814475bd714f5ca80f8bee 2024-11-28T01:45:10,138 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/table/74125ef68b814475bd714f5ca80f8bee, delay=983 ms 2024-11-28T01:45:10,138 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/table/74125ef68b814475bd714f5ca80f8bee, entries=2, sequenceid=11, filesize=5.3 K 2024-11-28T01:45:10,139 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=11, compaction requested=false 2024-11-28T01:45:10,140 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/info/c43f7e06ef6d46959cd48fcbd49ee361 2024-11-28T01:45:10,141 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/ns/645334fdce0c40b79695102805830ff7 2024-11-28T01:45:10,141 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/table/74125ef68b814475bd714f5ca80f8bee 2024-11-28T01:45:10,146 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-28T01:45:10,146 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T01:45:10,146 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T01:45:10,147 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732758310046Running coprocessor pre-close hooks at 1732758310046Disabling compacts and flushes for region at 1732758310046Disabling writes for close at 1732758310046Obtaining lock to block concurrent updates at 1732758310047 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732758310047Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732758310047Flushing stores of hbase:meta,,1.1588230740 at 1732758310047Flushing 1588230740/info: creating writer at 1732758310047Flushing 1588230740/info: appending metadata at 1732758310071 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732758310071Flushing 1588230740/ns: creating writer at 1732758310074 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732758310100 (+26 ms)Flushing 1588230740/ns: closing flushed file at 1732758310100Flushing 1588230740/table: creating writer at 1732758310104 (+4 ms)Flushing 1588230740/table: appending metadata at 1732758310127 (+23 ms)Flushing 1588230740/table: closing flushed file at 1732758310127Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c8b21d7: reopening flushed file at 1732758310131 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@250ff69a: reopening flushed file at 1732758310134 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a502e8a: reopening flushed file at 1732758310136 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=11, compaction requested=false at 1732758310139 (+3 ms)Writing region close event to WAL at 1732758310141 (+2 ms)Running coprocessor post-close hooks at 1732758310146 (+5 ms)Closed at 1732758310146 2024-11-28T01:45:10,147 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T01:45:10,246 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer(976): stopping server 9df3f25f5867,37707,1732758307259; all regions closed. 2024-11-28T01:45:10,248 DEBUG [RS:0;9df3f25f5867:37707 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/oldWALs 2024-11-28T01:45:10,248 INFO [RS:0;9df3f25f5867:37707 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9df3f25f5867%2C37707%2C1732758307259.meta:.meta(num 1732758308195) 2024-11-28T01:45:10,250 DEBUG [RS:0;9df3f25f5867:37707 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/oldWALs 2024-11-28T01:45:10,250 INFO [RS:0;9df3f25f5867:37707 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9df3f25f5867%2C37707%2C1732758307259:(num 1732758307825) 2024-11-28T01:45:10,250 DEBUG [RS:0;9df3f25f5867:37707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:10,250 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T01:45:10,250 INFO [RS:0;9df3f25f5867:37707 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T01:45:10,250 INFO [RS:0;9df3f25f5867:37707 {}] hbase.ChoreService(370): Chore service for: regionserver/9df3f25f5867:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-28T01:45:10,251 INFO [RS:0;9df3f25f5867:37707 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T01:45:10,251 INFO [regionserver/9df3f25f5867:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T01:45:10,251 INFO [RS:0;9df3f25f5867:37707 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37707 2024-11-28T01:45:10,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T01:45:10,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9df3f25f5867,37707,1732758307259 2024-11-28T01:45:10,256 INFO [RS:0;9df3f25f5867:37707 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T01:45:10,258 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9df3f25f5867,37707,1732758307259] 2024-11-28T01:45:10,259 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9df3f25f5867,37707,1732758307259 already deleted, retry=false 2024-11-28T01:45:10,259 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 9df3f25f5867,37707,1732758307259 on 9df3f25f5867,44337,1732758307209 2024-11-28T01:45:10,264 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 9df3f25f5867,37707,1732758307259, splitWal=true, meta=true 2024-11-28T01:45:10,266 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 9df3f25f5867,37707,1732758307259 (carryingMeta=true) 9df3f25f5867,37707,1732758307259/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@543fa427[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-28T01:45:10,267 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 9df3f25f5867,37707,1732758307259, splitWal=true, meta=true 2024-11-28T01:45:10,267 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 9df3f25f5867,37707,1732758307259, splitWal=true, meta=true, isMeta: true 2024-11-28T01:45:10,268 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,37707,1732758307259-splitting 2024-11-28T01:45:10,268 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,37707,1732758307259-splitting dir is empty, no logs to split. 2024-11-28T01:45:10,268 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9df3f25f5867,37707,1732758307259 WAL count=0, meta=true 2024-11-28T01:45:10,270 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,37707,1732758307259-splitting dir is empty, no logs to split. 2024-11-28T01:45:10,270 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9df3f25f5867,37707,1732758307259 WAL count=0, meta=true 2024-11-28T01:45:10,270 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 9df3f25f5867,37707,1732758307259 WAL splitting is done? wals=0, meta=true 2024-11-28T01:45:10,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T01:45:10,273 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T01:45:10,273 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-28T01:45:10,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:45:10,358 INFO [RS:0;9df3f25f5867:37707 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T01:45:10,358 INFO [RS:0;9df3f25f5867:37707 {}] regionserver.HRegionServer(1031): Exiting; stopping=9df3f25f5867,37707,1732758307259; zookeeper connection closed. 2024-11-28T01:45:10,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37707-0x1003bcd3a5e0001, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:45:10,358 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@10571c84 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@10571c84 2024-11-28T01:45:10,424 DEBUG [9df3f25f5867:44337 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-28T01:45:10,425 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9df3f25f5867,45483,1732758307295 2024-11-28T01:45:10,426 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9df3f25f5867,45483,1732758307295, state=OPENING 2024-11-28T01:45:10,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T01:45:10,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T01:45:10,428 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T01:45:10,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:10,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:10,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9df3f25f5867,45483,1732758307295}] 2024-11-28T01:45:10,543 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9df3f25f5867:0 server-side Connection retries=45 2024-11-28T01:45:10,543 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:10,543 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:10,543 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T01:45:10,543 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T01:45:10,543 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T01:45:10,543 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T01:45:10,543 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T01:45:10,544 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34701 2024-11-28T01:45:10,546 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34701 connecting to ZooKeeper ensemble=127.0.0.1:55202 2024-11-28T01:45:10,546 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:10,546 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:10,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:347010x0, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T01:45:10,550 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34701-0x1003bcd3a5e0003 connected 2024-11-28T01:45:10,550 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:34701-0x1003bcd3a5e0003, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-28T01:45:10,550 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-28T01:45:10,551 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T01:45:10,551 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/bucket.cache 2024-11-28T01:45:10,551 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-28T01:45:10,551 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-28T01:45:10,551 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-28T01:45:10,552 INFO [Thread-206 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-28T01:45:10,553 INFO [Thread-206 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-28T01:45:10,554 INFO [Thread-206 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-28T01:45:10,555 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-28T01:45:10,555 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T01:45:10,556 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:34701-0x1003bcd3a5e0003, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T01:45:10,558 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34701-0x1003bcd3a5e0003, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T01:45:10,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34701 2024-11-28T01:45:10,558 DEBUG [Thread-206 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-28T01:45:10,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34701 2024-11-28T01:45:10,559 INFO [Thread-206 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-28T01:45:10,559 INFO [Thread-206 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-28T01:45:10,559 INFO [Thread-206 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/bucket.persistence is complete. 2024-11-28T01:45:10,564 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34701 2024-11-28T01:45:10,564 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34701 2024-11-28T01:45:10,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34701 2024-11-28T01:45:10,571 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.HRegionServer(746): ClusterId : 239d6f20-9868-46de-9e93-c1fd463a2e4a 2024-11-28T01:45:10,571 DEBUG [RS:2;9df3f25f5867:34701 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T01:45:10,573 DEBUG [RS:2;9df3f25f5867:34701 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T01:45:10,574 DEBUG [RS:2;9df3f25f5867:34701 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T01:45:10,576 DEBUG [RS:2;9df3f25f5867:34701 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T01:45:10,577 DEBUG [RS:2;9df3f25f5867:34701 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f8089f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9df3f25f5867/172.17.0.2:0 2024-11-28T01:45:10,581 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T01:45:10,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57947, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T01:45:10,585 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T01:45:10,585 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T01:45:10,586 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T01:45:10,587 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9df3f25f5867%2C45483%2C1732758307295.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,45483,1732758307295, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/oldWALs, maxLogs=55 2024-11-28T01:45:10,593 DEBUG [RS:2;9df3f25f5867:34701 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;9df3f25f5867:34701 2024-11-28T01:45:10,593 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T01:45:10,593 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T01:45:10,593 DEBUG [RS:2;9df3f25f5867:34701 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T01:45:10,595 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.HRegionServer(2659): reportForDuty to master=9df3f25f5867,44337,1732758307209 with port=34701, startcode=1732758310542 2024-11-28T01:45:10,595 DEBUG [RS:2;9df3f25f5867:34701 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T01:45:10,597 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51857, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T01:45:10,598 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44337 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9df3f25f5867,34701,1732758310542 2024-11-28T01:45:10,598 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44337 {}] master.ServerManager(517): Registering regionserver=9df3f25f5867,34701,1732758310542 2024-11-28T01:45:10,600 DEBUG [RS:2;9df3f25f5867:34701 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c 2024-11-28T01:45:10,600 DEBUG [RS:2;9df3f25f5867:34701 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-28T01:45:10,600 DEBUG [RS:2;9df3f25f5867:34701 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T01:45:10,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T01:45:10,603 DEBUG [RS:2;9df3f25f5867:34701 {}] zookeeper.ZKUtil(111): regionserver:34701-0x1003bcd3a5e0003, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9df3f25f5867,34701,1732758310542 2024-11-28T01:45:10,603 WARN [RS:2;9df3f25f5867:34701 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T01:45:10,603 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9df3f25f5867,34701,1732758310542] 2024-11-28T01:45:10,603 INFO [RS:2;9df3f25f5867:34701 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T01:45:10,603 DEBUG [RS:2;9df3f25f5867:34701 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,34701,1732758310542 2024-11-28T01:45:10,609 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,45483,1732758307295/9df3f25f5867%2C45483%2C1732758307295.meta.1732758310588.meta 2024-11-28T01:45:10,610 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T01:45:10,612 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T01:45:10,612 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T01:45:10,612 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:45:10,613 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T01:45:10,613 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T01:45:10,613 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T01:45:10,613 INFO [RS:2;9df3f25f5867:34701 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T01:45:10,613 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T01:45:10,613 INFO [RS:2;9df3f25f5867:34701 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:10,613 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:45:10,613 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T01:45:10,613 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T01:45:10,613 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T01:45:10,614 INFO [RS:2;9df3f25f5867:34701 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T01:45:10,614 INFO [RS:2;9df3f25f5867:34701 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:10,614 DEBUG [RS:2;9df3f25f5867:34701 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:10,614 DEBUG [RS:2;9df3f25f5867:34701 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:10,614 DEBUG [RS:2;9df3f25f5867:34701 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:10,614 DEBUG [RS:2;9df3f25f5867:34701 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:10,614 DEBUG [RS:2;9df3f25f5867:34701 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:10,614 DEBUG [RS:2;9df3f25f5867:34701 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9df3f25f5867:0, corePoolSize=2, maxPoolSize=2 2024-11-28T01:45:10,614 DEBUG [RS:2;9df3f25f5867:34701 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:10,614 DEBUG [RS:2;9df3f25f5867:34701 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:10,614 DEBUG [RS:2;9df3f25f5867:34701 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:10,614 DEBUG [RS:2;9df3f25f5867:34701 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:10,614 DEBUG [RS:2;9df3f25f5867:34701 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:10,614 DEBUG [RS:2;9df3f25f5867:34701 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9df3f25f5867:0, corePoolSize=1, maxPoolSize=1 2024-11-28T01:45:10,615 DEBUG [RS:2;9df3f25f5867:34701 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9df3f25f5867:0, corePoolSize=3, maxPoolSize=3 2024-11-28T01:45:10,615 DEBUG [RS:2;9df3f25f5867:34701 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9df3f25f5867:0, corePoolSize=3, maxPoolSize=3 2024-11-28T01:45:10,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-28T01:45:10,615 INFO [RS:2;9df3f25f5867:34701 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:10,615 INFO [RS:2;9df3f25f5867:34701 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:10,615 INFO [RS:2;9df3f25f5867:34701 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:10,615 INFO [RS:2;9df3f25f5867:34701 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:10,615 INFO [RS:2;9df3f25f5867:34701 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:10,615 INFO [RS:2;9df3f25f5867:34701 {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,34701,1732758310542-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T01:45:10,616 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T01:45:10,616 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:10,617 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/info/c43f7e06ef6d46959cd48fcbd49ee361, delay=982 ms 2024-11-28T01:45:10,618 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/info/c43f7e06ef6d46959cd48fcbd49ee361 2024-11-28T01:45:10,619 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:10,619 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-28T01:45:10,619 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T01:45:10,619 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:10,621 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/ns/645334fdce0c40b79695102805830ff7, delay=966 ms 2024-11-28T01:45:10,623 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/ns/645334fdce0c40b79695102805830ff7 2024-11-28T01:45:10,623 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:10,623 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-28T01:45:10,624 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T01:45:10,624 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:10,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:10,625 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-28T01:45:10,625 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T01:45:10,625 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:10,629 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/table/74125ef68b814475bd714f5ca80f8bee, delay=931 ms 2024-11-28T01:45:10,630 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/table/74125ef68b814475bd714f5ca80f8bee 2024-11-28T01:45:10,630 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T01:45:10,630 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T01:45:10,631 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740 2024-11-28T01:45:10,632 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740 2024-11-28T01:45:10,632 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T01:45:10,632 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T01:45:10,633 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T01:45:10,634 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T01:45:10,634 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T01:45:10,634 INFO [RS:2;9df3f25f5867:34701 {}] hbase.ChoreService(168): Chore ScheduledChore name=9df3f25f5867,34701,1732758310542-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:10,634 INFO [RS:2;9df3f25f5867:34701 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:10,634 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.Replication(171): 9df3f25f5867,34701,1732758310542 started 2024-11-28T01:45:10,635 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72779521, jitterRate=0.08449937403202057}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T01:45:10,635 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T01:45:10,636 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732758310613Writing region info on filesystem at 1732758310613Initializing all the Stores at 1732758310614 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758310614Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758310615 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758310615Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732758310615Cleaning up temporary data from old regions at 1732758310632 (+17 ms)Running coprocessor post-open hooks at 1732758310635 (+3 ms)Region opened successfully at 1732758310636 (+1 ms) 2024-11-28T01:45:10,639 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732758310580 2024-11-28T01:45:10,643 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=9df3f25f5867,45483,1732758307295 2024-11-28T01:45:10,644 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9df3f25f5867,45483,1732758307295, state=OPEN 2024-11-28T01:45:10,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T01:45:10,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T01:45:10,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34701-0x1003bcd3a5e0003, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T01:45:10,646 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:10,646 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9df3f25f5867,45483,1732758307295 2024-11-28T01:45:10,646 DEBUG [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T01:45:10,647 INFO [RS_OPEN_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T01:45:10,647 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:10,647 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T01:45:10,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-28T01:45:10,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9df3f25f5867,45483,1732758307295 in 218 msec 2024-11-28T01:45:10,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-28T01:45:10,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 378 msec 2024-11-28T01:45:10,652 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 9df3f25f5867,37707,1732758307259 had 2 regions 2024-11-28T01:45:10,653 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 9df3f25f5867,37707,1732758307259, splitWal=true, meta=true, isMeta: false 2024-11-28T01:45:10,654 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,37707,1732758307259-splitting dir is empty, no logs to split. 2024-11-28T01:45:10,654 INFO [PEWorker-3 {}] master.SplitWALManager(105): 9df3f25f5867,37707,1732758307259 WAL count=0, meta=false 2024-11-28T01:45:10,655 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,37707,1732758307259-splitting dir is empty, no logs to split. 2024-11-28T01:45:10,655 INFO [PEWorker-3 {}] master.SplitWALManager(105): 9df3f25f5867,37707,1732758307259 WAL count=0, meta=false 2024-11-28T01:45:10,655 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 9df3f25f5867,37707,1732758307259 WAL splitting is done? wals=0, meta=false 2024-11-28T01:45:10,656 INFO [RS:2;9df3f25f5867:34701 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T01:45:10,656 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.HRegionServer(1482): Serving as 9df3f25f5867,34701,1732758310542, RpcServer on 9df3f25f5867/172.17.0.2:34701, sessionid=0x1003bcd3a5e0003 2024-11-28T01:45:10,656 DEBUG [RS:2;9df3f25f5867:34701 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T01:45:10,656 DEBUG [RS:2;9df3f25f5867:34701 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9df3f25f5867,34701,1732758310542 2024-11-28T01:45:10,656 DEBUG [RS:2;9df3f25f5867:34701 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9df3f25f5867,34701,1732758310542' 2024-11-28T01:45:10,656 DEBUG [RS:2;9df3f25f5867:34701 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T01:45:10,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732758308341, region=04c80cae9a2e31b89a0a10577aea4215, ASSIGN}] 2024-11-28T01:45:10,657 DEBUG [RS:2;9df3f25f5867:34701 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T01:45:10,658 DEBUG [RS:2;9df3f25f5867:34701 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T01:45:10,658 DEBUG [RS:2;9df3f25f5867:34701 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T01:45:10,658 DEBUG [RS:2;9df3f25f5867:34701 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9df3f25f5867,34701,1732758310542 2024-11-28T01:45:10,658 DEBUG [RS:2;9df3f25f5867:34701 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9df3f25f5867,34701,1732758310542' 2024-11-28T01:45:10,658 DEBUG [RS:2;9df3f25f5867:34701 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T01:45:10,658 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732758308341, region=04c80cae9a2e31b89a0a10577aea4215, ASSIGN 2024-11-28T01:45:10,659 DEBUG [RS:2;9df3f25f5867:34701 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T01:45:10,660 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732758308341, region=04c80cae9a2e31b89a0a10577aea4215, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-28T01:45:10,660 DEBUG [RS:2;9df3f25f5867:34701 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T01:45:10,660 INFO [RS:2;9df3f25f5867:34701 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T01:45:10,660 INFO [RS:2;9df3f25f5867:34701 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T01:45:10,761 INFO [RS:2;9df3f25f5867:34701 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T01:45:10,762 INFO [RS:2;9df3f25f5867:34701 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9df3f25f5867%2C34701%2C1732758310542, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,34701,1732758310542, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/oldWALs, maxLogs=55 2024-11-28T01:45:10,788 INFO [RS:2;9df3f25f5867:34701 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/WALs/9df3f25f5867,34701,1732758310542/9df3f25f5867%2C34701%2C1732758310542.1732758310763 2024-11-28T01:45:10,789 DEBUG [RS:2;9df3f25f5867:34701 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T01:45:10,810 DEBUG [9df3f25f5867:44337 {}] balancer.BalancerClusterState(204): Hosts are {9df3f25f5867=0} racks are {/default-rack=0} 2024-11-28T01:45:10,810 DEBUG [9df3f25f5867:44337 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T01:45:10,810 DEBUG [9df3f25f5867:44337 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T01:45:10,810 DEBUG [9df3f25f5867:44337 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T01:45:10,810 DEBUG [9df3f25f5867:44337 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T01:45:10,810 INFO [9df3f25f5867:44337 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T01:45:10,810 INFO [9df3f25f5867:44337 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T01:45:10,810 DEBUG [9df3f25f5867:44337 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T01:45:10,811 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=04c80cae9a2e31b89a0a10577aea4215, regionState=OPENING, regionLocation=9df3f25f5867,34701,1732758310542 2024-11-28T01:45:10,812 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9df3f25f5867:37707 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9df3f25f5867/172.17.0.2:37707 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T01:45:10,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,37707,1732758307259, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,37707,1732758307259, seqNum=-1, error=java.net.ConnectException: Call to address=9df3f25f5867:37707 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9df3f25f5867/172.17.0.2:37707 2024-11-28T01:45:10,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,37707,1732758307259, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-28T01:45:10,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,37707,1732758307259, seqNum=-1 from cache 2024-11-28T01:45:10,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address 9df3f25f5867:37707 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9df3f25f5867/172.17.0.2:37707 2024-11-28T01:45:10,918 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T01:45:10,918 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9df3f25f5867,45483,1732758307295, seqNum=-1] 2024-11-28T01:45:10,918 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T01:45:10,920 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58975, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T01:45:10,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732758308341, region=04c80cae9a2e31b89a0a10577aea4215, ASSIGN because future has completed 2024-11-28T01:45:10,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04c80cae9a2e31b89a0a10577aea4215, server=9df3f25f5867,34701,1732758310542}] 2024-11-28T01:45:11,075 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T01:45:11,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41549, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T01:45:11,079 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:11,080 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 04c80cae9a2e31b89a0a10577aea4215, NAME => 'testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215.', STARTKEY => '', ENDKEY => ''} 2024-11-28T01:45:11,080 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732758308341 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:11,080 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T01:45:11,081 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:11,081 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:11,081 INFO [StoreOpener-04c80cae9a2e31b89a0a10577aea4215-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:11,082 INFO [StoreOpener-04c80cae9a2e31b89a0a10577aea4215-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04c80cae9a2e31b89a0a10577aea4215 columnFamilyName family 2024-11-28T01:45:11,082 DEBUG [StoreOpener-04c80cae9a2e31b89a0a10577aea4215-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T01:45:11,084 DEBUG [StoreFileOpener-04c80cae9a2e31b89a0a10577aea4215-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/family/93048ea5a43a4ce795eca170b65d7f1e, delay=922 ms 2024-11-28T01:45:11,085 DEBUG [StoreOpener-04c80cae9a2e31b89a0a10577aea4215-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/family/93048ea5a43a4ce795eca170b65d7f1e 2024-11-28T01:45:11,086 INFO [StoreOpener-04c80cae9a2e31b89a0a10577aea4215-1 {}] regionserver.HStore(327): Store=04c80cae9a2e31b89a0a10577aea4215/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T01:45:11,086 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:11,086 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:11,087 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:11,087 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:11,087 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:11,088 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:11,089 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 04c80cae9a2e31b89a0a10577aea4215; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60087655, jitterRate=-0.10462416708469391}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T01:45:11,089 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:11,090 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 04c80cae9a2e31b89a0a10577aea4215: Running coprocessor pre-open hook at 1732758311081Writing region info on filesystem at 1732758311081Initializing all the Stores at 1732758311081Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732758311081Cleaning up temporary data from old regions at 1732758311087 (+6 ms)Running coprocessor post-open hooks at 1732758311089 (+2 ms)Region opened successfully at 1732758311090 (+1 ms) 2024-11-28T01:45:11,091 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215., pid=11, masterSystemTime=1732758311075 2024-11-28T01:45:11,094 DEBUG [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:11,094 INFO [RS_OPEN_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:11,095 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=04c80cae9a2e31b89a0a10577aea4215, regionState=OPEN, openSeqNum=10, regionLocation=9df3f25f5867,34701,1732758310542 2024-11-28T01:45:11,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04c80cae9a2e31b89a0a10577aea4215, server=9df3f25f5867,34701,1732758310542 because future has completed 2024-11-28T01:45:11,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-28T01:45:11,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 04c80cae9a2e31b89a0a10577aea4215, server=9df3f25f5867,34701,1732758310542 in 175 msec 2024-11-28T01:45:11,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-28T01:45:11,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732758308341, region=04c80cae9a2e31b89a0a10577aea4215, ASSIGN in 443 msec 2024-11-28T01:45:11,102 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 9df3f25f5867,37707,1732758307259 after splitting done 2024-11-28T01:45:11,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 9df3f25f5867,37707,1732758307259, splitWal=true, meta=true in 843 msec 2024-11-28T01:45:11,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-28T01:45:11,157 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T01:45:11,157 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:45:11,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:11,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:11,157 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T01:45:11,158 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-28T01:45:11,158 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1137076648, stopped=false 2024-11-28T01:45:11,158 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9df3f25f5867,44337,1732758307209 2024-11-28T01:45:11,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T01:45:11,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34701-0x1003bcd3a5e0003, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T01:45:11,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T01:45:11,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:11,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34701-0x1003bcd3a5e0003, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:11,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:11,160 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T01:45:11,160 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T01:45:11,160 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:45:11,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:11,161 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T01:45:11,161 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9df3f25f5867,45483,1732758307295' ***** 2024-11-28T01:45:11,161 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34701-0x1003bcd3a5e0003, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T01:45:11,161 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T01:45:11,161 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9df3f25f5867,34701,1732758310542' ***** 2024-11-28T01:45:11,161 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T01:45:11,161 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T01:45:11,161 INFO [RS:1;9df3f25f5867:45483 {}] bucket.BucketCache(1738): Disabling cache 2024-11-28T01:45:11,161 INFO [RS:1;9df3f25f5867:45483 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/bucket.persistence 2024-11-28T01:45:11,161 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-28T01:45:11,161 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-28T01:45:11,162 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-28T01:45:11,162 INFO [RS:1;9df3f25f5867:45483 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-28T01:45:11,162 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-28T01:45:11,162 INFO [RS:2;9df3f25f5867:34701 {}] bucket.BucketCache(1738): Disabling cache 2024-11-28T01:45:11,163 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-28T01:45:11,163 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-28T01:45:11,163 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-28T01:45:11,163 INFO [RS:2;9df3f25f5867:34701 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/174125f7-67da-32d4-925d-63271e41c80b/bucket.persistence 2024-11-28T01:45:11,163 INFO [RS:2;9df3f25f5867:34701 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-28T01:45:11,163 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-28T01:45:11,172 DEBUG [RS:1;9df3f25f5867:45483 {}] bucket.BucketCache(1487): Thread RS:1;9df3f25f5867:45483 started persisting bucket cache to file 2024-11-28T01:45:11,172 DEBUG [RS:1;9df3f25f5867:45483 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T01:45:11,172 DEBUG [RS:1;9df3f25f5867:45483 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T01:45:11,173 DEBUG [RS:2;9df3f25f5867:34701 {}] bucket.BucketCache(1487): Thread RS:2;9df3f25f5867:34701 started persisting bucket cache to file 2024-11-28T01:45:11,173 DEBUG [RS:2;9df3f25f5867:34701 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T01:45:11,173 DEBUG [RS:2;9df3f25f5867:34701 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T01:45:11,176 DEBUG [RS:1;9df3f25f5867:45483 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-28T01:45:11,176 DEBUG [RS:1;9df3f25f5867:45483 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732758311172 2024-11-28T01:45:11,176 DEBUG [RS:1;9df3f25f5867:45483 {}] bucket.BucketCache(1509): Thread RS:1;9df3f25f5867:45483 finished persisting bucket cache to file, renaming 2024-11-28T01:45:11,176 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T01:45:11,176 INFO [RS:1;9df3f25f5867:45483 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T01:45:11,176 INFO [RS:1;9df3f25f5867:45483 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T01:45:11,176 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.HRegionServer(959): stopping server 9df3f25f5867,45483,1732758307295 2024-11-28T01:45:11,176 INFO [RS:1;9df3f25f5867:45483 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T01:45:11,176 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T01:45:11,177 INFO [RS:1;9df3f25f5867:45483 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9df3f25f5867:45483. 2024-11-28T01:45:11,177 DEBUG [RS:1;9df3f25f5867:45483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:45:11,177 DEBUG [RS:2;9df3f25f5867:34701 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-28T01:45:11,177 DEBUG [RS:1;9df3f25f5867:45483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:11,177 DEBUG [RS:2;9df3f25f5867:34701 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732758311173 2024-11-28T01:45:11,177 DEBUG [RS:2;9df3f25f5867:34701 {}] bucket.BucketCache(1509): Thread RS:2;9df3f25f5867:34701 finished persisting bucket cache to file, renaming 2024-11-28T01:45:11,177 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T01:45:11,177 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T01:45:11,177 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T01:45:11,177 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T01:45:11,177 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T01:45:11,177 INFO [RS:2;9df3f25f5867:34701 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T01:45:11,177 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T01:45:11,177 INFO [RS:2;9df3f25f5867:34701 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T01:45:11,177 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-28T01:45:11,177 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.HRegionServer(3091): Received CLOSE for 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:11,177 DEBUG [RS:1;9df3f25f5867:45483 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-28T01:45:11,178 DEBUG [RS:1;9df3f25f5867:45483 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T01:45:11,178 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T01:45:11,178 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.HRegionServer(959): stopping server 9df3f25f5867,34701,1732758310542 2024-11-28T01:45:11,178 INFO [RS:2;9df3f25f5867:34701 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T01:45:11,178 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T01:45:11,178 INFO [RS:2;9df3f25f5867:34701 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;9df3f25f5867:34701. 2024-11-28T01:45:11,178 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T01:45:11,178 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T01:45:11,178 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 04c80cae9a2e31b89a0a10577aea4215, disabling compactions & flushes 2024-11-28T01:45:11,178 DEBUG [RS:2;9df3f25f5867:34701 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T01:45:11,178 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T01:45:11,178 DEBUG [RS:2;9df3f25f5867:34701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:11,178 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:11,178 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T01:45:11,178 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:11,178 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T01:45:11,178 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-28T01:45:11,178 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. after waiting 0 ms 2024-11-28T01:45:11,178 DEBUG [RS:2;9df3f25f5867:34701 {}] regionserver.HRegionServer(1325): Online Regions={04c80cae9a2e31b89a0a10577aea4215=testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215.} 2024-11-28T01:45:11,178 DEBUG [RS:2;9df3f25f5867:34701 {}] regionserver.HRegionServer(1351): Waiting on 04c80cae9a2e31b89a0a10577aea4215 2024-11-28T01:45:11,178 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:11,178 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-28T01:45:11,179 DEBUG [StoreFileCloser-04c80cae9a2e31b89a0a10577aea4215-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/family/93048ea5a43a4ce795eca170b65d7f1e 2024-11-28T01:45:11,180 INFO [regionserver/9df3f25f5867:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T01:45:11,185 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/default/testBlockEvictionOnGracefulStop1732758308341/04c80cae9a2e31b89a0a10577aea4215/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-28T01:45:11,185 INFO [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:11,185 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 04c80cae9a2e31b89a0a10577aea4215: Waiting for close lock at 1732758311178Running coprocessor pre-close hooks at 1732758311178Disabling compacts and flushes for region at 1732758311178Disabling writes for close at 1732758311178Writing region close event to WAL at 1732758311179 (+1 ms)Running coprocessor post-close hooks at 1732758311185 (+6 ms)Closed at 1732758311185 2024-11-28T01:45:11,185 DEBUG [RS_CLOSE_REGION-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215. 2024-11-28T01:45:11,190 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 10c3eb824da74b808e77bb479f2acb78 2024-11-28T01:45:11,205 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/info/10c3eb824da74b808e77bb479f2acb78 is 209, key is testBlockEvictionOnGracefulStop1732758308341,,1732758308345.04c80cae9a2e31b89a0a10577aea4215./info:regioninfo/1732758311094/Put/seqid=0 2024-11-28T01:45:11,206 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/info/10c3eb824da74b808e77bb479f2acb78 2024-11-28T01:45:11,208 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/.tmp/info/10c3eb824da74b808e77bb479f2acb78 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/info/10c3eb824da74b808e77bb479f2acb78 2024-11-28T01:45:11,209 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/info/10c3eb824da74b808e77bb479f2acb78, delay=966 ms 2024-11-28T01:45:11,210 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/info/10c3eb824da74b808e77bb479f2acb78, entries=8, sequenceid=19, filesize=6.6 K 2024-11-28T01:45:11,211 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 33ms, sequenceid=19, compaction requested=false 2024-11-28T01:45:11,212 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/info/c43f7e06ef6d46959cd48fcbd49ee361 2024-11-28T01:45:11,212 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/info/10c3eb824da74b808e77bb479f2acb78 2024-11-28T01:45:11,213 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/ns/645334fdce0c40b79695102805830ff7 2024-11-28T01:45:11,214 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/table/74125ef68b814475bd714f5ca80f8bee 2024-11-28T01:45:11,216 INFO [regionserver/9df3f25f5867:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T01:45:11,219 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-28T01:45:11,219 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T01:45:11,219 INFO [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T01:45:11,219 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732758311178Running coprocessor pre-close hooks at 1732758311178Disabling compacts and flushes for region at 1732758311178Disabling writes for close at 1732758311178Obtaining lock to block concurrent updates at 1732758311178Preparing flush snapshotting stores in 1588230740 at 1732758311178Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732758311179 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732758311179Flushing 1588230740/info: creating writer at 1732758311179Flushing 1588230740/info: appending metadata at 1732758311204 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732758311204Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d4fc493: reopening flushed file at 1732758311208 (+4 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 33ms, sequenceid=19, compaction requested=false at 1732758311211 (+3 ms)Writing region close event to WAL at 1732758311214 (+3 ms)Running coprocessor post-close hooks at 1732758311219 (+5 ms)Closed at 1732758311219 2024-11-28T01:45:11,220 DEBUG [RS_CLOSE_META-regionserver/9df3f25f5867:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T01:45:11,378 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.HRegionServer(976): stopping server 9df3f25f5867,45483,1732758307295; all regions closed. 2024-11-28T01:45:11,378 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.HRegionServer(976): stopping server 9df3f25f5867,34701,1732758310542; all regions closed. 2024-11-28T01:45:11,380 DEBUG [RS:1;9df3f25f5867:45483 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/oldWALs 2024-11-28T01:45:11,380 INFO [RS:1;9df3f25f5867:45483 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9df3f25f5867%2C45483%2C1732758307295.meta:.meta(num 1732758310588) 2024-11-28T01:45:11,380 DEBUG [RS:2;9df3f25f5867:34701 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/oldWALs 2024-11-28T01:45:11,381 INFO [RS:2;9df3f25f5867:34701 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9df3f25f5867%2C34701%2C1732758310542:(num 1732758310763) 2024-11-28T01:45:11,381 DEBUG [RS:2;9df3f25f5867:34701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:11,381 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T01:45:11,381 INFO [RS:2;9df3f25f5867:34701 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T01:45:11,381 INFO [RS:2;9df3f25f5867:34701 {}] hbase.ChoreService(370): Chore service for: regionserver/9df3f25f5867:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-28T01:45:11,381 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T01:45:11,381 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T01:45:11,381 INFO [regionserver/9df3f25f5867:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T01:45:11,381 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T01:45:11,381 INFO [RS:2;9df3f25f5867:34701 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T01:45:11,382 INFO [RS:2;9df3f25f5867:34701 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34701 2024-11-28T01:45:11,382 DEBUG [RS:1;9df3f25f5867:45483 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/oldWALs 2024-11-28T01:45:11,382 INFO [RS:1;9df3f25f5867:45483 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9df3f25f5867%2C45483%2C1732758307295:(num 1732758307825) 2024-11-28T01:45:11,383 DEBUG [RS:1;9df3f25f5867:45483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T01:45:11,383 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T01:45:11,383 INFO [RS:1;9df3f25f5867:45483 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T01:45:11,383 INFO [RS:1;9df3f25f5867:45483 {}] hbase.ChoreService(370): Chore service for: regionserver/9df3f25f5867:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-28T01:45:11,383 INFO [RS:1;9df3f25f5867:45483 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T01:45:11,383 INFO [regionserver/9df3f25f5867:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T01:45:11,383 INFO [RS:1;9df3f25f5867:45483 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45483 2024-11-28T01:45:11,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34701-0x1003bcd3a5e0003, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9df3f25f5867,34701,1732758310542 2024-11-28T01:45:11,384 INFO [RS:2;9df3f25f5867:34701 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T01:45:11,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T01:45:11,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9df3f25f5867,45483,1732758307295 2024-11-28T01:45:11,386 INFO [RS:1;9df3f25f5867:45483 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T01:45:11,386 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9df3f25f5867,34701,1732758310542] 2024-11-28T01:45:11,388 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9df3f25f5867,34701,1732758310542 already deleted, retry=false 2024-11-28T01:45:11,388 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9df3f25f5867,34701,1732758310542 expired; onlineServers=1 2024-11-28T01:45:11,388 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9df3f25f5867,45483,1732758307295] 2024-11-28T01:45:11,389 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9df3f25f5867,45483,1732758307295 already deleted, retry=false 2024-11-28T01:45:11,390 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9df3f25f5867,45483,1732758307295 expired; onlineServers=0 2024-11-28T01:45:11,390 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9df3f25f5867,44337,1732758307209' ***** 2024-11-28T01:45:11,390 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-28T01:45:11,390 INFO [M:0;9df3f25f5867:44337 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T01:45:11,390 INFO [M:0;9df3f25f5867:44337 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T01:45:11,390 DEBUG [M:0;9df3f25f5867:44337 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-28T01:45:11,390 DEBUG [M:0;9df3f25f5867:44337 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-28T01:45:11,390 DEBUG [master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.large.0-1732758307566 {}] cleaner.HFileCleaner(306): Exit Thread[master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.large.0-1732758307566,5,FailOnTimeoutGroup] 2024-11-28T01:45:11,390 DEBUG [master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.small.0-1732758307567 {}] cleaner.HFileCleaner(306): Exit Thread[master/9df3f25f5867:0:becomeActiveMaster-HFileCleaner.small.0-1732758307567,5,FailOnTimeoutGroup] 2024-11-28T01:45:11,390 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-28T01:45:11,390 INFO [M:0;9df3f25f5867:44337 {}] hbase.ChoreService(370): Chore service for: master/9df3f25f5867:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-28T01:45:11,390 INFO [M:0;9df3f25f5867:44337 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T01:45:11,391 DEBUG [M:0;9df3f25f5867:44337 {}] master.HMaster(1795): Stopping service threads 2024-11-28T01:45:11,391 INFO [M:0;9df3f25f5867:44337 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-28T01:45:11,391 INFO [M:0;9df3f25f5867:44337 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T01:45:11,391 INFO [M:0;9df3f25f5867:44337 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T01:45:11,391 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-28T01:45:11,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-28T01:45:11,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T01:45:11,392 DEBUG [M:0;9df3f25f5867:44337 {}] zookeeper.ZKUtil(347): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-28T01:45:11,392 WARN [M:0;9df3f25f5867:44337 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-28T01:45:11,392 INFO [M:0;9df3f25f5867:44337 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/.lastflushedseqids 2024-11-28T01:45:11,396 INFO [M:0;9df3f25f5867:44337 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T01:45:11,396 INFO [M:0;9df3f25f5867:44337 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-28T01:45:11,396 DEBUG [M:0;9df3f25f5867:44337 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T01:45:11,396 INFO [M:0;9df3f25f5867:44337 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:45:11,396 DEBUG [M:0;9df3f25f5867:44337 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:45:11,397 DEBUG [M:0;9df3f25f5867:44337 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T01:45:11,397 DEBUG [M:0;9df3f25f5867:44337 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:45:11,397 INFO [M:0;9df3f25f5867:44337 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.51 KB heapSize=49.91 KB 2024-11-28T01:45:11,407 DEBUG [M:0;9df3f25f5867:44337 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 266bf99c22d54d3cb39b1af3c5ab54b8 2024-11-28T01:45:11,429 DEBUG [M:0;9df3f25f5867:44337 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/266bf99c22d54d3cb39b1af3c5ab54b8 is 82, key is hbase:meta,,1/info:regioninfo/1732758310643/Put/seqid=0 2024-11-28T01:45:11,429 INFO [M:0;9df3f25f5867:44337 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/266bf99c22d54d3cb39b1af3c5ab54b8 2024-11-28T01:45:11,439 DEBUG [M:0;9df3f25f5867:44337 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 982df1ac71254dbfb9a6db086f661177 2024-11-28T01:45:11,454 DEBUG [M:0;9df3f25f5867:44337 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/982df1ac71254dbfb9a6db086f661177 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732758308723/Put/seqid=0 2024-11-28T01:45:11,455 INFO [M:0;9df3f25f5867:44337 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.28 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/982df1ac71254dbfb9a6db086f661177 2024-11-28T01:45:11,463 DEBUG [M:0;9df3f25f5867:44337 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 67bbf08906b44758a96a8bd7fa25fa14 2024-11-28T01:45:11,478 DEBUG [M:0;9df3f25f5867:44337 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/67bbf08906b44758a96a8bd7fa25fa14 is 69, key is 9df3f25f5867,34701,1732758310542/rs:state/1732758310598/Put/seqid=0 2024-11-28T01:45:11,478 INFO [M:0;9df3f25f5867:44337 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/67bbf08906b44758a96a8bd7fa25fa14 2024-11-28T01:45:11,480 INFO [M:0;9df3f25f5867:44337 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 67bbf08906b44758a96a8bd7fa25fa14 2024-11-28T01:45:11,480 DEBUG [M:0;9df3f25f5867:44337 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/266bf99c22d54d3cb39b1af3c5ab54b8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/266bf99c22d54d3cb39b1af3c5ab54b8 2024-11-28T01:45:11,482 INFO [M:0;9df3f25f5867:44337 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/266bf99c22d54d3cb39b1af3c5ab54b8, entries=14, sequenceid=102, filesize=5.9 K 2024-11-28T01:45:11,482 DEBUG [M:0;9df3f25f5867:44337 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/982df1ac71254dbfb9a6db086f661177 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/982df1ac71254dbfb9a6db086f661177 2024-11-28T01:45:11,484 INFO [M:0;9df3f25f5867:44337 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/982df1ac71254dbfb9a6db086f661177, entries=11, sequenceid=102, filesize=6.5 K 2024-11-28T01:45:11,484 DEBUG [M:0;9df3f25f5867:44337 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/67bbf08906b44758a96a8bd7fa25fa14 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/67bbf08906b44758a96a8bd7fa25fa14 2024-11-28T01:45:11,486 INFO [M:0;9df3f25f5867:44337 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 67bbf08906b44758a96a8bd7fa25fa14 2024-11-28T01:45:11,486 INFO [M:0;9df3f25f5867:44337 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c95cd160-771b-153a-bbd0-6e2b9f7688ae/dd75ba1a-ee67-2c0b-7b05-d965f56d872c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/67bbf08906b44758a96a8bd7fa25fa14, entries=3, sequenceid=102, filesize=5.3 K 2024-11-28T01:45:11,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34701-0x1003bcd3a5e0003, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:45:11,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34701-0x1003bcd3a5e0003, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:45:11,487 INFO [RS:2;9df3f25f5867:34701 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T01:45:11,487 INFO [RS:2;9df3f25f5867:34701 {}] regionserver.HRegionServer(1031): Exiting; stopping=9df3f25f5867,34701,1732758310542; zookeeper connection closed. 2024-11-28T01:45:11,487 INFO [M:0;9df3f25f5867:44337 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=102, compaction requested=false 2024-11-28T01:45:11,487 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5a8715a3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5a8715a3 2024-11-28T01:45:11,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:45:11,488 INFO [RS:1;9df3f25f5867:45483 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T01:45:11,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45483-0x1003bcd3a5e0002, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:45:11,488 INFO [RS:1;9df3f25f5867:45483 {}] regionserver.HRegionServer(1031): Exiting; stopping=9df3f25f5867,45483,1732758307295; zookeeper connection closed. 2024-11-28T01:45:11,488 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@780b026c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@780b026c 2024-11-28T01:45:11,488 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-28T01:45:11,489 INFO [M:0;9df3f25f5867:44337 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T01:45:11,489 DEBUG [M:0;9df3f25f5867:44337 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732758311396Disabling compacts and flushes for region at 1732758311396Disabling writes for close at 1732758311397 (+1 ms)Obtaining lock to block concurrent updates at 1732758311397Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732758311397Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40460, getHeapSize=51048, getOffHeapSize=0, getCellsCount=127 at 1732758311397Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732758311398 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732758311398Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732758311428 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732758311428Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732758311432 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732758311453 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732758311453Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732758311456 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732758311477 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732758311477Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29c87c39: reopening flushed file at 1732758311480 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49beea98: reopening flushed file at 1732758311482 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32a5a0c1: reopening flushed file at 1732758311484 (+2 ms)Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=102, compaction requested=false at 1732758311487 (+3 ms)Writing region close event to WAL at 1732758311489 (+2 ms)Closed at 1732758311489 2024-11-28T01:45:11,490 INFO [M:0;9df3f25f5867:44337 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-28T01:45:11,490 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T01:45:11,490 INFO [M:0;9df3f25f5867:44337 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44337 2024-11-28T01:45:11,490 INFO [M:0;9df3f25f5867:44337 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T01:45:11,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:45:11,593 INFO [M:0;9df3f25f5867:44337 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T01:45:11,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44337-0x1003bcd3a5e0000, quorum=127.0.0.1:55202, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T01:45:11,596 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-28T01:45:11,597 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-28T01:45:11,597 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T01:45:11,604 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 49) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 274), ProcessCount=11 (was 11), AvailableMemoryMB=7393 (was 7523)