2024-11-14 02:46:45,968 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-14 02:46:45,990 main DEBUG Took 0.019747 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 02:46:45,990 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 02:46:45,990 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 02:46:45,991 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 02:46:45,992 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:46:46,001 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 02:46:46,016 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:46:46,017 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:46:46,018 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:46:46,018 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:46:46,019 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:46:46,020 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:46:46,021 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:46:46,022 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:46:46,022 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:46:46,023 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:46:46,024 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:46:46,024 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:46:46,025 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:46:46,025 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:46:46,026 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:46:46,026 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:46:46,026 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:46:46,027 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:46:46,027 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:46:46,027 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:46:46,028 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:46:46,028 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:46:46,029 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:46:46,029 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 02:46:46,029 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:46:46,030 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 02:46:46,031 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 02:46:46,033 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 02:46:46,034 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 02:46:46,035 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 02:46:46,037 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 02:46:46,037 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 02:46:46,046 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 02:46:46,048 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 02:46:46,050 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 02:46:46,050 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 02:46:46,051 main DEBUG createAppenders(={Console}) 2024-11-14 02:46:46,051 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-14 02:46:46,052 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-14 02:46:46,052 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-14 02:46:46,052 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 02:46:46,053 main DEBUG OutputStream closed 2024-11-14 02:46:46,053 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 02:46:46,053 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 02:46:46,053 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-14 02:46:46,136 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 02:46:46,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 02:46:46,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 02:46:46,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 02:46:46,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 02:46:46,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 02:46:46,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 02:46:46,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 02:46:46,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 02:46:46,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 02:46:46,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 02:46:46,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 02:46:46,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 02:46:46,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 02:46:46,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 02:46:46,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 02:46:46,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 02:46:46,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 02:46:46,152 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 02:46:46,152 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-14 02:46:46,152 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 02:46:46,153 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-14T02:46:46,167 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-14 02:46:46,170 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 02:46:46,170 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T02:46:46,461 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679 2024-11-14T02:46:46,483 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=487, ProcessCount=11, AvailableMemoryMB=11647 2024-11-14T02:46:46,564 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T02:46:46,645 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:46,660 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/cluster_bc1ffc6d-56f4-87ca-2d78-23cab9bb77c0, deleteOnExit=true 2024-11-14T02:46:46,790 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/cluster_bc1ffc6d-56f4-87ca-2d78-23cab9bb77c0/zookeeper_0, clientPort=54067, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/cluster_bc1ffc6d-56f4-87ca-2d78-23cab9bb77c0/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/cluster_bc1ffc6d-56f4-87ca-2d78-23cab9bb77c0/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T02:46:46,803 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54067 2024-11-14T02:46:46,804 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:46,900 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414 with version=8 2024-11-14T02:46:46,901 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/hbase-staging 2024-11-14T02:46:47,005 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T02:46:47,283 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e5dc8b5cf5e9:0 server-side Connection retries=45 2024-11-14T02:46:47,295 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:47,297 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:47,307 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T02:46:47,307 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:47,308 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T02:46:47,487 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T02:46:47,569 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T02:46:47,613 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T02:46:47,623 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T02:46:47,627 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T02:46:47,673 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 15552 (auto-detected) 2024-11-14T02:46:47,675 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T02:46:47,700 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41647 2024-11-14T02:46:47,731 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41647 connecting to ZooKeeper ensemble=127.0.0.1:54067 2024-11-14T02:46:47,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:416470x0, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T02:46:47,870 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41647-0x101373ce89c0000 connected 2024-11-14T02:46:47,966 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:47,967 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:47,975 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:46:47,980 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414, hbase.cluster.distributed=false 2024-11-14T02:46:48,012 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T02:46:48,021 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41647 2024-11-14T02:46:48,022 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41647 2024-11-14T02:46:48,022 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41647 2024-11-14T02:46:48,023 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41647 2024-11-14T02:46:48,023 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41647 2024-11-14T02:46:48,114 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e5dc8b5cf5e9:0 server-side Connection retries=45 2024-11-14T02:46:48,116 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:48,116 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:48,117 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T02:46:48,117 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:48,117 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T02:46:48,120 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T02:46:48,124 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T02:46:48,125 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42581 2024-11-14T02:46:48,127 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42581 connecting to ZooKeeper ensemble=127.0.0.1:54067 2024-11-14T02:46:48,128 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:48,129 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:48,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:425810x0, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T02:46:48,182 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:425810x0, quorum=127.0.0.1:54067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:46:48,182 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42581-0x101373ce89c0001 connected 2024-11-14T02:46:48,191 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T02:46:48,205 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/bucket.cache 2024-11-14T02:46:48,211 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-14T02:46:48,212 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-14T02:46:48,214 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-14T02:46:48,215 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-14T02:46:48,216 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-14T02:46:48,223 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-14T02:46:48,227 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-14T02:46:48,227 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/bucket.persistence is complete. 2024-11-14T02:46:48,229 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T02:46:48,232 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T02:46:48,239 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T02:46:48,247 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42581 2024-11-14T02:46:48,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42581 2024-11-14T02:46:48,255 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42581 2024-11-14T02:46:48,260 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42581 2024-11-14T02:46:48,261 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42581 2024-11-14T02:46:48,279 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e5dc8b5cf5e9:0 server-side Connection retries=45 2024-11-14T02:46:48,279 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:48,279 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:48,280 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T02:46:48,280 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:48,280 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T02:46:48,280 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T02:46:48,281 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T02:46:48,282 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41247 2024-11-14T02:46:48,284 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41247 connecting to ZooKeeper ensemble=127.0.0.1:54067 2024-11-14T02:46:48,284 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:48,285 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:48,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:412470x0, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T02:46:48,300 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:412470x0, quorum=127.0.0.1:54067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:46:48,300 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41247-0x101373ce89c0002 connected 2024-11-14T02:46:48,301 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T02:46:48,304 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/bucket.cache 2024-11-14T02:46:48,304 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-14T02:46:48,305 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-14T02:46:48,305 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-14T02:46:48,306 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-14T02:46:48,306 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-14T02:46:48,307 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-14T02:46:48,307 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T02:46:48,308 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-14T02:46:48,308 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/bucket.persistence is complete. 2024-11-14T02:46:48,309 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T02:46:48,311 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T02:46:48,312 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41247 2024-11-14T02:46:48,312 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41247 2024-11-14T02:46:48,315 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41247 2024-11-14T02:46:48,316 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41247 2024-11-14T02:46:48,320 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41247 2024-11-14T02:46:48,336 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e5dc8b5cf5e9:41647 2024-11-14T02:46:48,337 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e5dc8b5cf5e9,41647,1731552407099 2024-11-14T02:46:48,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:46:48,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:46:48,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:46:48,355 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e5dc8b5cf5e9,41647,1731552407099 2024-11-14T02:46:48,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T02:46:48,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:48,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T02:46:48,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:48,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:48,395 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T02:46:48,396 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e5dc8b5cf5e9,41647,1731552407099 from backup master directory 2024-11-14T02:46:48,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:46:48,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e5dc8b5cf5e9,41647,1731552407099 2024-11-14T02:46:48,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:46:48,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:46:48,405 WARN [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T02:46:48,405 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e5dc8b5cf5e9,41647,1731552407099 2024-11-14T02:46:48,408 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T02:46:48,409 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T02:46:48,419 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/hbase.id] with ID: d1e86a32-73b7-40ce-9e75-f72d636b1e3d 2024-11-14T02:46:48,419 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/.tmp/hbase.id 2024-11-14T02:46:48,425 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/hbase.id] 2024-11-14T02:46:48,456 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:48,461 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T02:46:48,482 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-14T02:46:48,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:48,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:48,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:48,520 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:46:48,523 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T02:46:48,562 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T02:46:48,567 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T02:46:48,620 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store 2024-11-14T02:46:48,639 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T02:46:48,642 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:48,644 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T02:46:48,644 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:48,644 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:48,646 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T02:46:48,646 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:48,646 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:48,647 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731552408644Disabling compacts and flushes for region at 1731552408644Disabling writes for close at 1731552408646 (+2 ms)Writing region close event to WAL at 1731552408646Closed at 1731552408646 2024-11-14T02:46:48,647 WARN [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/.initializing 2024-11-14T02:46:48,648 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/WALs/e5dc8b5cf5e9,41647,1731552407099 2024-11-14T02:46:48,662 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T02:46:48,686 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e5dc8b5cf5e9%2C41647%2C1731552407099, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/WALs/e5dc8b5cf5e9,41647,1731552407099, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/oldWALs, maxLogs=10 2024-11-14T02:46:48,738 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/WALs/e5dc8b5cf5e9,41647,1731552407099/e5dc8b5cf5e9%2C41647%2C1731552407099.1731552408693 2024-11-14T02:46:48,741 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T02:46:48,742 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:46:48,742 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:48,747 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:48,749 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:48,791 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:48,831 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T02:46:48,835 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:48,837 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:48,837 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:48,842 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T02:46:48,842 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:48,843 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:48,843 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:48,847 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T02:46:48,848 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:48,848 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:48,848 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:48,853 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T02:46:48,853 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:48,854 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:48,854 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:48,857 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:48,858 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:48,861 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:48,861 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:48,864 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T02:46:48,866 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:48,872 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:46:48,873 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69043093, jitterRate=0.02882225811481476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T02:46:48,881 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731552408763Initializing all the Stores at 1731552408763Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552408764 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552408765 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552408766 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552408766Cleaning up temporary data from old regions at 1731552408861 (+95 ms)Region opened successfully at 1731552408880 (+19 ms) 2024-11-14T02:46:48,882 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T02:46:48,916 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d87cc4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e5dc8b5cf5e9/172.17.0.2:0 2024-11-14T02:46:48,957 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T02:46:48,973 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T02:46:48,973 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T02:46:48,977 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T02:46:48,978 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T02:46:48,985 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-14T02:46:48,985 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T02:46:49,015 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T02:46:49,027 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T02:46:49,078 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T02:46:49,081 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T02:46:49,083 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T02:46:49,096 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T02:46:49,098 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T02:46:49,102 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T02:46:49,109 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T02:46:49,111 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T02:46:49,119 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T02:46:49,141 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T02:46:49,151 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T02:46:49,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T02:46:49,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T02:46:49,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T02:46:49,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:49,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:49,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:49,176 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e5dc8b5cf5e9,41647,1731552407099, sessionid=0x101373ce89c0000, setting cluster-up flag (Was=false) 2024-11-14T02:46:49,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:49,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:49,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:49,246 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T02:46:49,248 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e5dc8b5cf5e9,41647,1731552407099 2024-11-14T02:46:49,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:49,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:49,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:49,298 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T02:46:49,301 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e5dc8b5cf5e9,41647,1731552407099 2024-11-14T02:46:49,308 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T02:46:49,324 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HRegionServer(746): ClusterId : d1e86a32-73b7-40ce-9e75-f72d636b1e3d 2024-11-14T02:46:49,324 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HRegionServer(746): ClusterId : d1e86a32-73b7-40ce-9e75-f72d636b1e3d 2024-11-14T02:46:49,327 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T02:46:49,327 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T02:46:49,343 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T02:46:49,343 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T02:46:49,343 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T02:46:49,343 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T02:46:49,353 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T02:46:49,353 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T02:46:49,353 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa531fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e5dc8b5cf5e9/172.17.0.2:0 2024-11-14T02:46:49,353 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b0247a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e5dc8b5cf5e9/172.17.0.2:0 2024-11-14T02:46:49,368 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;e5dc8b5cf5e9:41247 2024-11-14T02:46:49,368 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e5dc8b5cf5e9:42581 2024-11-14T02:46:49,371 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T02:46:49,371 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T02:46:49,371 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T02:46:49,371 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T02:46:49,371 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T02:46:49,372 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T02:46:49,374 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HRegionServer(2659): reportForDuty to master=e5dc8b5cf5e9,41647,1731552407099 with port=41247, startcode=1731552408278 2024-11-14T02:46:49,374 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HRegionServer(2659): reportForDuty to master=e5dc8b5cf5e9,41647,1731552407099 with port=42581, startcode=1731552408075 2024-11-14T02:46:49,385 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T02:46:49,386 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T02:46:49,386 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T02:46:49,396 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T02:46:49,405 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T02:46:49,409 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e5dc8b5cf5e9,41647,1731552407099 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T02:46:49,417 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e5dc8b5cf5e9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T02:46:49,417 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e5dc8b5cf5e9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T02:46:49,417 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e5dc8b5cf5e9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T02:46:49,417 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e5dc8b5cf5e9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T02:46:49,418 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e5dc8b5cf5e9:0, corePoolSize=10, maxPoolSize=10 2024-11-14T02:46:49,418 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,418 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e5dc8b5cf5e9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T02:46:49,419 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,420 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731552439420 2024-11-14T02:46:49,424 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T02:46:49,425 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T02:46:49,425 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T02:46:49,425 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T02:46:49,429 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T02:46:49,431 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T02:46:49,432 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T02:46:49,432 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T02:46:49,437 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:49,437 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T02:46:49,440 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,447 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T02:46:49,449 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T02:46:49,449 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T02:46:49,453 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T02:46:49,453 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T02:46:49,456 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.large.0-1731552409455,5,FailOnTimeoutGroup] 2024-11-14T02:46:49,457 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.small.0-1731552409456,5,FailOnTimeoutGroup] 2024-11-14T02:46:49,457 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,457 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T02:46:49,458 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,459 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,461 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T02:46:49,462 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414 2024-11-14T02:46:49,462 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34803, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T02:46:49,462 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54345, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T02:46:49,470 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41647 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e5dc8b5cf5e9,42581,1731552408075 2024-11-14T02:46:49,470 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:49,472 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41647 {}] master.ServerManager(517): Registering regionserver=e5dc8b5cf5e9,42581,1731552408075 2024-11-14T02:46:49,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-14T02:46:49,477 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T02:46:49,477 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:49,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:49,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-14T02:46:49,481 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41647 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e5dc8b5cf5e9,41247,1731552408278 2024-11-14T02:46:49,482 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41647 {}] master.ServerManager(517): Registering regionserver=e5dc8b5cf5e9,41247,1731552408278 2024-11-14T02:46:49,483 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T02:46:49,483 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:49,484 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:49,484 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-14T02:46:49,486 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414 2024-11-14T02:46:49,486 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-14T02:46:49,486 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T02:46:49,487 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414 2024-11-14T02:46:49,488 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-14T02:46:49,488 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T02:46:49,489 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T02:46:49,490 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:49,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:49,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-14T02:46:49,497 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T02:46:49,498 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:49,498 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:49,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T02:46:49,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740 2024-11-14T02:46:49,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740 2024-11-14T02:46:49,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T02:46:49,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T02:46:49,501 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T02:46:49,502 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T02:46:49,511 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:46:49,511 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74284997, jitterRate=0.10693271458148956}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T02:46:49,517 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731552409470Initializing all the Stores at 1731552409470Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552409470Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552409471 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552409472 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552409472Cleaning up temporary data from old regions at 1731552409501 (+29 ms)Region opened successfully at 1731552409517 (+16 ms) 2024-11-14T02:46:49,518 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T02:46:49,518 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T02:46:49,518 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T02:46:49,518 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T02:46:49,518 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T02:46:49,520 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T02:46:49,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731552409518Disabling compacts and flushes for region at 1731552409518Disabling writes for close at 1731552409518Writing region close event to WAL at 1731552409519 (+1 ms)Closed at 1731552409520 (+1 ms) 2024-11-14T02:46:49,522 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T02:46:49,522 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T02:46:49,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T02:46:49,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T02:46:49,533 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] zookeeper.ZKUtil(111): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e5dc8b5cf5e9,41247,1731552408278 2024-11-14T02:46:49,533 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] zookeeper.ZKUtil(111): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e5dc8b5cf5e9,42581,1731552408075 2024-11-14T02:46:49,534 WARN [RS:1;e5dc8b5cf5e9:41247 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T02:46:49,534 WARN [RS:0;e5dc8b5cf5e9:42581 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T02:46:49,534 INFO [RS:0;e5dc8b5cf5e9:42581 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T02:46:49,534 INFO [RS:1;e5dc8b5cf5e9:41247 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T02:46:49,534 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,42581,1731552408075 2024-11-14T02:46:49,534 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,41247,1731552408278 2024-11-14T02:46:49,536 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e5dc8b5cf5e9,41247,1731552408278] 2024-11-14T02:46:49,536 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e5dc8b5cf5e9,42581,1731552408075] 2024-11-14T02:46:49,542 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T02:46:49,544 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T02:46:49,565 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T02:46:49,565 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T02:46:49,584 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T02:46:49,584 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T02:46:49,588 INFO [RS:1;e5dc8b5cf5e9:41247 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T02:46:49,588 INFO [RS:0;e5dc8b5cf5e9:42581 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T02:46:49,588 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,589 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,590 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T02:46:49,590 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T02:46:49,595 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T02:46:49,595 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T02:46:49,597 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,597 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,597 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,597 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,597 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,597 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,598 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,598 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,598 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,598 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,598 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,598 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,598 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e5dc8b5cf5e9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T02:46:49,598 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e5dc8b5cf5e9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T02:46:49,598 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,598 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,598 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,598 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,599 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,599 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,599 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,599 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,599 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,599 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,599 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,599 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:49,599 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e5dc8b5cf5e9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T02:46:49,599 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e5dc8b5cf5e9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T02:46:49,600 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T02:46:49,600 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T02:46:49,601 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,601 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,601 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,601 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,602 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,602 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,602 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,602 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,602 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,602 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,602 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,42581,1731552408075-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T02:46:49,602 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,41247,1731552408278-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T02:46:49,625 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T02:46:49,628 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,42581,1731552408075-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,628 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,628 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.Replication(171): e5dc8b5cf5e9,42581,1731552408075 started 2024-11-14T02:46:49,633 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T02:46:49,633 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,41247,1731552408278-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,633 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,633 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.Replication(171): e5dc8b5cf5e9,41247,1731552408278 started 2024-11-14T02:46:49,649 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,650 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HRegionServer(1482): Serving as e5dc8b5cf5e9,42581,1731552408075, RpcServer on e5dc8b5cf5e9/172.17.0.2:42581, sessionid=0x101373ce89c0001 2024-11-14T02:46:49,651 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T02:46:49,651 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e5dc8b5cf5e9,42581,1731552408075 2024-11-14T02:46:49,651 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5dc8b5cf5e9,42581,1731552408075' 2024-11-14T02:46:49,651 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T02:46:49,652 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T02:46:49,653 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T02:46:49,653 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T02:46:49,654 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e5dc8b5cf5e9,42581,1731552408075 2024-11-14T02:46:49,654 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5dc8b5cf5e9,42581,1731552408075' 2024-11-14T02:46:49,654 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T02:46:49,654 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T02:46:49,655 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T02:46:49,655 INFO [RS:0;e5dc8b5cf5e9:42581 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T02:46:49,655 INFO [RS:0;e5dc8b5cf5e9:42581 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T02:46:49,657 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:49,658 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HRegionServer(1482): Serving as e5dc8b5cf5e9,41247,1731552408278, RpcServer on e5dc8b5cf5e9/172.17.0.2:41247, sessionid=0x101373ce89c0002 2024-11-14T02:46:49,658 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T02:46:49,658 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e5dc8b5cf5e9,41247,1731552408278 2024-11-14T02:46:49,658 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5dc8b5cf5e9,41247,1731552408278' 2024-11-14T02:46:49,658 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T02:46:49,659 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T02:46:49,660 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T02:46:49,660 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T02:46:49,660 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e5dc8b5cf5e9,41247,1731552408278 2024-11-14T02:46:49,660 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5dc8b5cf5e9,41247,1731552408278' 2024-11-14T02:46:49,660 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T02:46:49,660 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T02:46:49,661 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T02:46:49,661 INFO [RS:1;e5dc8b5cf5e9:41247 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T02:46:49,661 INFO [RS:1;e5dc8b5cf5e9:41247 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T02:46:49,695 WARN [e5dc8b5cf5e9:41647 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-14T02:46:49,761 INFO [RS:0;e5dc8b5cf5e9:42581 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T02:46:49,762 INFO [RS:1;e5dc8b5cf5e9:41247 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T02:46:49,762 INFO [RS:0;e5dc8b5cf5e9:42581 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e5dc8b5cf5e9%2C42581%2C1731552408075, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,42581,1731552408075, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/oldWALs, maxLogs=55 2024-11-14T02:46:49,763 INFO [RS:1;e5dc8b5cf5e9:41247 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e5dc8b5cf5e9%2C41247%2C1731552408278, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,41247,1731552408278, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/oldWALs, maxLogs=55 2024-11-14T02:46:49,784 INFO [RS:1;e5dc8b5cf5e9:41247 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,41247,1731552408278/e5dc8b5cf5e9%2C41247%2C1731552408278.1731552409766 2024-11-14T02:46:49,784 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T02:46:49,785 INFO [RS:0;e5dc8b5cf5e9:42581 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,42581,1731552408075/e5dc8b5cf5e9%2C42581%2C1731552408075.1731552409766 2024-11-14T02:46:49,785 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T02:46:49,949 DEBUG [e5dc8b5cf5e9:41647 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-14T02:46:49,964 DEBUG [e5dc8b5cf5e9:41647 {}] balancer.BalancerClusterState(204): Hosts are {e5dc8b5cf5e9=0} racks are {/default-rack=0} 2024-11-14T02:46:49,968 DEBUG [e5dc8b5cf5e9:41647 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T02:46:49,968 DEBUG [e5dc8b5cf5e9:41647 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T02:46:49,968 DEBUG [e5dc8b5cf5e9:41647 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T02:46:49,968 DEBUG [e5dc8b5cf5e9:41647 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T02:46:49,968 INFO [e5dc8b5cf5e9:41647 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T02:46:49,968 INFO [e5dc8b5cf5e9:41647 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T02:46:49,968 DEBUG [e5dc8b5cf5e9:41647 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T02:46:49,976 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e5dc8b5cf5e9,41247,1731552408278 2024-11-14T02:46:49,981 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e5dc8b5cf5e9,41247,1731552408278, state=OPENING 2024-11-14T02:46:50,035 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T02:46:50,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:50,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:50,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:50,047 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:46:50,047 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:46:50,047 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:46:50,050 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T02:46:50,053 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e5dc8b5cf5e9,41247,1731552408278}] 2024-11-14T02:46:50,231 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T02:46:50,234 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44647, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T02:46:50,244 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T02:46:50,244 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T02:46:50,245 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T02:46:50,247 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e5dc8b5cf5e9%2C41247%2C1731552408278.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,41247,1731552408278, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/oldWALs, maxLogs=55 2024-11-14T02:46:50,269 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,41247,1731552408278/e5dc8b5cf5e9%2C41247%2C1731552408278.meta.1731552410249.meta 2024-11-14T02:46:50,269 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T02:46:50,269 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:46:50,271 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T02:46:50,274 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T02:46:50,278 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T02:46:50,282 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T02:46:50,283 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:50,283 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T02:46:50,283 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T02:46:50,285 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-14T02:46:50,286 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T02:46:50,287 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:50,287 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:50,287 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-14T02:46:50,288 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T02:46:50,288 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:50,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:50,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-14T02:46:50,289 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T02:46:50,289 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:50,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:50,290 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-14T02:46:50,290 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T02:46:50,291 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:50,291 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:50,291 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T02:46:50,292 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740 2024-11-14T02:46:50,293 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740 2024-11-14T02:46:50,294 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T02:46:50,294 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T02:46:50,294 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T02:46:50,295 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T02:46:50,296 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60227862, jitterRate=-0.10253491997718811}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T02:46:50,296 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T02:46:50,299 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731552410284Writing region info on filesystem at 1731552410284Initializing all the Stores at 1731552410285 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552410285Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552410285Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552410285Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552410285Cleaning up temporary data from old regions at 1731552410294 (+9 ms)Running coprocessor post-open hooks at 1731552410296 (+2 ms)Region opened successfully at 1731552410298 (+2 ms) 2024-11-14T02:46:50,304 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731552410225 2024-11-14T02:46:50,316 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T02:46:50,317 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T02:46:50,318 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e5dc8b5cf5e9,41247,1731552408278 2024-11-14T02:46:50,320 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e5dc8b5cf5e9,41247,1731552408278, state=OPEN 2024-11-14T02:46:50,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T02:46:50,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T02:46:50,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T02:46:50,331 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:46:50,331 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:46:50,331 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:46:50,331 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e5dc8b5cf5e9,41247,1731552408278 2024-11-14T02:46:50,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T02:46:50,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e5dc8b5cf5e9,41247,1731552408278 in 278 msec 2024-11-14T02:46:50,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T02:46:50,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 807 msec 2024-11-14T02:46:50,343 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T02:46:50,343 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T02:46:50,367 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:46:50,369 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,41247,1731552408278, seqNum=-1] 2024-11-14T02:46:50,392 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:46:50,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36657, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:46:50,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0720 sec 2024-11-14T02:46:50,413 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731552410413, completionTime=-1 2024-11-14T02:46:50,416 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-14T02:46:50,417 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T02:46:50,444 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T02:46:50,445 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731552470445 2024-11-14T02:46:50,445 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731552530445 2024-11-14T02:46:50,445 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-14T02:46:50,447 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-14T02:46:50,454 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,41647,1731552407099-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:50,454 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,41647,1731552407099-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:50,455 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,41647,1731552407099-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:50,457 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e5dc8b5cf5e9:41647, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:50,457 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:50,459 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:50,464 DEBUG [master/e5dc8b5cf5e9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T02:46:50,486 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.080sec 2024-11-14T02:46:50,488 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T02:46:50,489 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T02:46:50,490 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T02:46:50,491 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T02:46:50,491 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T02:46:50,492 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,41647,1731552407099-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T02:46:50,493 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,41647,1731552407099-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T02:46:50,494 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T02:46:50,496 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T02:46:50,496 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,41647,1731552407099-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:50,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71985066, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:50,540 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T02:46:50,540 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T02:46:50,546 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,41647,-1 for getting cluster id 2024-11-14T02:46:50,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:46:50,561 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1e86a32-73b7-40ce-9e75-f72d636b1e3d' 2024-11-14T02:46:50,564 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:46:50,564 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1e86a32-73b7-40ce-9e75-f72d636b1e3d" 2024-11-14T02:46:50,567 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fa220d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:50,568 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,41647,-1] 2024-11-14T02:46:50,571 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:46:50,573 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:50,574 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:46:50,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd60c17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:50,578 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:46:50,589 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,41247,1731552408278, seqNum=-1] 2024-11-14T02:46:50,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:46:50,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:46:50,625 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e5dc8b5cf5e9,41647,1731552407099 2024-11-14T02:46:50,631 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T02:46:50,636 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e5dc8b5cf5e9,41647,1731552407099 2024-11-14T02:46:50,638 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6ea750a6 2024-11-14T02:46:50,642 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T02:46:50,645 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T02:46:50,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731552410625', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:46:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731552410625 2024-11-14T02:46:50,660 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731552410625 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T02:46:50,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731552410625" procId is: 4 2024-11-14T02:46:50,663 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:50,665 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731552410625 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T02:46:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:46:50,682 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731552410625-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3669efda7c21025c952e3223a054c5d8, NAME => 'testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731552410625', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414 2024-11-14T02:46:50,688 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731552410625-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:50,688 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731552410625-pool-0 {}] regionserver.HRegion(1722): Closing 3669efda7c21025c952e3223a054c5d8, disabling compactions & flushes 2024-11-14T02:46:50,688 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731552410625-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:50,688 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731552410625-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:50,688 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731552410625-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. after waiting 0 ms 2024-11-14T02:46:50,688 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731552410625-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:50,688 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731552410625-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:50,688 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731552410625-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3669efda7c21025c952e3223a054c5d8: Waiting for close lock at 1731552410688Disabling compacts and flushes for region at 1731552410688Disabling writes for close at 1731552410688Writing region close event to WAL at 1731552410688Closed at 1731552410688 2024-11-14T02:46:50,690 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731552410625 execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T02:46:50,694 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731552410690"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731552410690"}]},"ts":"1731552410690"} 2024-11-14T02:46:50,699 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T02:46:50,701 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731552410625 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T02:46:50,704 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731552410625","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552410701"}]},"ts":"1731552410701"} 2024-11-14T02:46:50,708 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731552410625, state=ENABLING in hbase:meta 2024-11-14T02:46:50,709 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(204): Hosts are {e5dc8b5cf5e9=0} racks are {/default-rack=0} 2024-11-14T02:46:50,709 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T02:46:50,709 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T02:46:50,709 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T02:46:50,709 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T02:46:50,709 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T02:46:50,709 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T02:46:50,709 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T02:46:50,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731552410625, region=3669efda7c21025c952e3223a054c5d8, ASSIGN}] 2024-11-14T02:46:50,713 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731552410625, region=3669efda7c21025c952e3223a054c5d8, ASSIGN 2024-11-14T02:46:50,714 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731552410625, region=3669efda7c21025c952e3223a054c5d8, ASSIGN; state=OFFLINE, location=e5dc8b5cf5e9,41247,1731552408278; forceNewPlan=false, retain=false 2024-11-14T02:46:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:46:50,866 INFO [e5dc8b5cf5e9:41647 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T02:46:50,866 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3669efda7c21025c952e3223a054c5d8, regionState=OPENING, regionLocation=e5dc8b5cf5e9,41247,1731552408278 2024-11-14T02:46:50,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731552410625, region=3669efda7c21025c952e3223a054c5d8, ASSIGN because future has completed 2024-11-14T02:46:50,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3669efda7c21025c952e3223a054c5d8, server=e5dc8b5cf5e9,41247,1731552408278}] 2024-11-14T02:46:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:46:51,028 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:51,028 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3669efda7c21025c952e3223a054c5d8, NAME => 'testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:46:51,029 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731552410625 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:51,029 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:51,029 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:51,029 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:51,030 INFO [StoreOpener-3669efda7c21025c952e3223a054c5d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:51,035 INFO [StoreOpener-3669efda7c21025c952e3223a054c5d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3669efda7c21025c952e3223a054c5d8 columnFamilyName family 2024-11-14T02:46:51,036 DEBUG [StoreOpener-3669efda7c21025c952e3223a054c5d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:51,036 INFO [StoreOpener-3669efda7c21025c952e3223a054c5d8-1 {}] regionserver.HStore(327): Store=3669efda7c21025c952e3223a054c5d8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:51,036 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:51,038 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:51,038 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:51,038 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:51,038 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:51,039 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:51,044 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:46:51,045 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3669efda7c21025c952e3223a054c5d8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60881376, jitterRate=-0.09279680252075195}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T02:46:51,045 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:51,046 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3669efda7c21025c952e3223a054c5d8: Running coprocessor pre-open hook at 1731552411030Writing region info on filesystem at 1731552411030Initializing all the Stores at 1731552411030Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552411030Cleaning up temporary data from old regions at 1731552411038 (+8 ms)Running coprocessor post-open hooks at 1731552411045 (+7 ms)Region opened successfully at 1731552411046 (+1 ms) 2024-11-14T02:46:51,048 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8., pid=6, masterSystemTime=1731552411024 2024-11-14T02:46:51,051 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:51,051 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:51,052 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3669efda7c21025c952e3223a054c5d8, regionState=OPEN, openSeqNum=2, regionLocation=e5dc8b5cf5e9,41247,1731552408278 2024-11-14T02:46:51,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3669efda7c21025c952e3223a054c5d8, server=e5dc8b5cf5e9,41247,1731552408278 because future has completed 2024-11-14T02:46:51,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T02:46:51,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3669efda7c21025c952e3223a054c5d8, server=e5dc8b5cf5e9,41247,1731552408278 in 186 msec 2024-11-14T02:46:51,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T02:46:51,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731552410625, region=3669efda7c21025c952e3223a054c5d8, ASSIGN in 349 msec 2024-11-14T02:46:51,064 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731552410625 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T02:46:51,065 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731552410625","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552411064"}]},"ts":"1731552411064"} 2024-11-14T02:46:51,068 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731552410625, state=ENABLED in hbase:meta 2024-11-14T02:46:51,069 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731552410625 execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T02:46:51,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731552410625 in 416 msec 2024-11-14T02:46:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:46:51,303 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731552410625 get assigned. Timeout = 60000ms 2024-11-14T02:46:51,303 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731552410625 completed 2024-11-14T02:46:51,306 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T02:46:51,313 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731552410625 assigned to meta. Checking AM states. 2024-11-14T02:46:51,313 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T02:46:51,314 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731552410625 assigned. 2024-11-14T02:46:51,325 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731552410625', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8., hostname=e5dc8b5cf5e9,41247,1731552408278, seqNum=2] 2024-11-14T02:46:51,338 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3669efda7c21025c952e3223a054c5d8 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-14T02:46:51,365 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 13a696c4f20b4bb687f7300ca4a9abaf 2024-11-14T02:46:51,412 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/.tmp/family/13a696c4f20b4bb687f7300ca4a9abaf is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-14T02:46:51,418 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/.tmp/family/13a696c4f20b4bb687f7300ca4a9abaf 2024-11-14T02:46:51,453 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/.tmp/family/13a696c4f20b4bb687f7300ca4a9abaf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/family/13a696c4f20b4bb687f7300ca4a9abaf 2024-11-14T02:46:51,455 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/family/13a696c4f20b4bb687f7300ca4a9abaf, delay=900 ms 2024-11-14T02:46:51,457 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/family/13a696c4f20b4bb687f7300ca4a9abaf, entries=2, sequenceid=6, filesize=4.9 K 2024-11-14T02:46:51,464 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 3669efda7c21025c952e3223a054c5d8 in 125ms, sequenceid=6, compaction requested=false 2024-11-14T02:46:51,465 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731552410625' 2024-11-14T02:46:51,468 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3669efda7c21025c952e3223a054c5d8: 2024-11-14T02:46:52,357 DEBUG [hfile-prefetch-1731552411455 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 13a696c4f20b4bb687f7300ca4a9abaf_0. 2024-11-14T02:46:52,363 DEBUG [hfile-prefetch-1731552411455 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 13a696c4f20b4bb687f7300ca4a9abaf_117. 2024-11-14T02:46:52,364 DEBUG [hfile-prefetch-1731552411455 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/family/13a696c4f20b4bb687f7300ca4a9abaf, with total blocks 2, and data blocks 1 2024-11-14T02:46:52,365 DEBUG [hfile-prefetch-1731552411455 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-14T02:46:52,365 DEBUG [hfile-prefetch-1731552411455 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/family/13a696c4f20b4bb687f7300ca4a9abaf, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-14T02:46:52,366 DEBUG [hfile-prefetch-1731552411455 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/family/13a696c4f20b4bb687f7300ca4a9abaf. Will sleep 100ms and try the verification again. 2024-11-14T02:46:52,466 DEBUG [hfile-prefetch-1731552411455 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/family/13a696c4f20b4bb687f7300ca4a9abaf, with total blocks 2, and data blocks 1 2024-11-14T02:46:52,466 DEBUG [hfile-prefetch-1731552411455 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-14T02:46:52,466 DEBUG [hfile-prefetch-1731552411455 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/family/13a696c4f20b4bb687f7300ca4a9abaf, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-14T02:46:52,466 DEBUG [hfile-prefetch-1731552411455 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/family/13a696c4f20b4bb687f7300ca4a9abaf. Will sleep 100ms and try the verification again. 2024-11-14T02:46:52,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=3669efda7c21025c952e3223a054c5d8, source=e5dc8b5cf5e9,41247,1731552408278, destination=e5dc8b5cf5e9,42581,1731552408075, warming up region on e5dc8b5cf5e9,42581,1731552408075 2024-11-14T02:46:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T02:46:52,488 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=3669efda7c21025c952e3223a054c5d8, source=e5dc8b5cf5e9,41247,1731552408278, destination=e5dc8b5cf5e9,42581,1731552408075, running balancer 2024-11-14T02:46:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731552410625, region=3669efda7c21025c952e3223a054c5d8, REOPEN/MOVE 2024-11-14T02:46:52,489 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731552410625, region=3669efda7c21025c952e3223a054c5d8, REOPEN/MOVE 2024-11-14T02:46:52,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60865, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T02:46:52,490 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3669efda7c21025c952e3223a054c5d8, regionState=CLOSING, regionLocation=e5dc8b5cf5e9,41247,1731552408278 2024-11-14T02:46:52,491 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42581 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42581 {}] regionserver.HRegion(7855): Warmup {ENCODED => 3669efda7c21025c952e3223a054c5d8, NAME => 'testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:46:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42581 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:52,492 INFO [StoreOpener-3669efda7c21025c952e3223a054c5d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:52,493 INFO [StoreOpener-3669efda7c21025c952e3223a054c5d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3669efda7c21025c952e3223a054c5d8 columnFamilyName family 2024-11-14T02:46:52,493 DEBUG [StoreOpener-3669efda7c21025c952e3223a054c5d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:52,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731552410625, region=3669efda7c21025c952e3223a054c5d8, REOPEN/MOVE because future has completed 2024-11-14T02:46:52,494 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-14T02:46:52,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3669efda7c21025c952e3223a054c5d8, server=e5dc8b5cf5e9,41247,1731552408278}] 2024-11-14T02:46:52,497 DEBUG [StoreOpener-3669efda7c21025c952e3223a054c5d8-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/family/13a696c4f20b4bb687f7300ca4a9abaf 2024-11-14T02:46:52,497 INFO [StoreOpener-3669efda7c21025c952e3223a054c5d8-1 {}] regionserver.HStore(327): Store=3669efda7c21025c952e3223a054c5d8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42581 {}] regionserver.HRegion(1722): Closing 3669efda7c21025c952e3223a054c5d8, disabling compactions & flushes 2024-11-14T02:46:52,498 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42581 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42581 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42581 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. after waiting 0 ms 2024-11-14T02:46:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42581 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:52,499 DEBUG [StoreFileCloser-3669efda7c21025c952e3223a054c5d8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/family/13a696c4f20b4bb687f7300ca4a9abaf 2024-11-14T02:46:52,499 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42581 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42581 {}] regionserver.HRegion(1676): Region close journal for 3669efda7c21025c952e3223a054c5d8: Waiting for close lock at 1731552412497Disabling compacts and flushes for region at 1731552412497Disabling writes for close at 1731552412498 (+1 ms)Writing region close event to WAL at 1731552412499 (+1 ms)Closed at 1731552412499 2024-11-14T02:46:52,501 WARN [hfile-prefetch-1731552411455 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/family/13a696c4f20b4bb687f7300ca4a9abaf, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2322) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 10 more 2024-11-14T02:46:52,501 DEBUG [hfile-prefetch-1731552411455 {}] hfile.PrefetchExecutor(121): Prefetch completed for 13a696c4f20b4bb687f7300ca4a9abaf 2024-11-14T02:46:52,509 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-14T02:46:52,510 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-14T02:46:52,510 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T02:46:52,510 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T02:46:52,525 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-14T02:46:52,525 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-14T02:46:52,525 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T02:46:52,525 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T02:46:52,529 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-14T02:46:52,529 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731552412525 2024-11-14T02:46:52,529 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-14T02:46:52,531 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-14T02:46:52,532 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731552412510 2024-11-14T02:46:52,532 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-14T02:46:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41647 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-14T02:46:52,652 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:52,653 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-14T02:46:52,653 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 3669efda7c21025c952e3223a054c5d8, disabling compactions & flushes 2024-11-14T02:46:52,653 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:52,653 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:52,654 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. after waiting 0 ms 2024-11-14T02:46:52,654 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:52,656 DEBUG [StoreFileCloser-3669efda7c21025c952e3223a054c5d8-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file 13a696c4f20b4bb687f7300ca4a9abaf, starting offset: 0, end offset: 9223372036854775807 2024-11-14T02:46:52,657 DEBUG [StoreFileCloser-3669efda7c21025c952e3223a054c5d8-family-1 {}] bucket.BucketCache(857): removed key 13a696c4f20b4bb687f7300ca4a9abaf_0 from back map with offset lock 2094080 in the evict process 2024-11-14T02:46:52,663 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-14T02:46:52,665 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:52,665 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 3669efda7c21025c952e3223a054c5d8: Waiting for close lock at 1731552412653Running coprocessor pre-close hooks at 1731552412653Disabling compacts and flushes for region at 1731552412653Disabling writes for close at 1731552412654 (+1 ms)Writing region close event to WAL at 1731552412658 (+4 ms)Running coprocessor post-close hooks at 1731552412663 (+5 ms)Closed at 1731552412664 (+1 ms) 2024-11-14T02:46:52,665 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 3669efda7c21025c952e3223a054c5d8 move to e5dc8b5cf5e9,42581,1731552408075 record at close sequenceid=6 2024-11-14T02:46:52,670 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:52,670 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3669efda7c21025c952e3223a054c5d8, regionState=CLOSED 2024-11-14T02:46:52,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3669efda7c21025c952e3223a054c5d8, server=e5dc8b5cf5e9,41247,1731552408278 because future has completed 2024-11-14T02:46:52,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T02:46:52,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 3669efda7c21025c952e3223a054c5d8, server=e5dc8b5cf5e9,41247,1731552408278 in 180 msec 2024-11-14T02:46:52,677 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731552410625, region=3669efda7c21025c952e3223a054c5d8, REOPEN/MOVE; state=CLOSED, location=e5dc8b5cf5e9,42581,1731552408075; forceNewPlan=false, retain=false 2024-11-14T02:46:52,729 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-14T02:46:52,729 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-14T02:46:52,729 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T02:46:52,729 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T02:46:52,732 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-14T02:46:52,732 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-14T02:46:52,732 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-14T02:46:52,732 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731552412729 2024-11-14T02:46:52,732 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T02:46:52,732 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-14T02:46:52,732 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T02:46:52,736 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-14T02:46:52,737 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731552412732 2024-11-14T02:46:52,737 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-14T02:46:52,828 INFO [e5dc8b5cf5e9:41647 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T02:46:52,828 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3669efda7c21025c952e3223a054c5d8, regionState=OPENING, regionLocation=e5dc8b5cf5e9,42581,1731552408075 2024-11-14T02:46:52,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731552410625, region=3669efda7c21025c952e3223a054c5d8, REOPEN/MOVE because future has completed 2024-11-14T02:46:52,832 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3669efda7c21025c952e3223a054c5d8, server=e5dc8b5cf5e9,42581,1731552408075}] 2024-11-14T02:46:52,990 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:52,991 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 3669efda7c21025c952e3223a054c5d8, NAME => 'testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:46:52,992 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731552410625 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:52,992 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:52,992 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:52,992 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:52,993 INFO [StoreOpener-3669efda7c21025c952e3223a054c5d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:52,995 INFO [StoreOpener-3669efda7c21025c952e3223a054c5d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3669efda7c21025c952e3223a054c5d8 columnFamilyName family 2024-11-14T02:46:52,995 DEBUG [StoreOpener-3669efda7c21025c952e3223a054c5d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:52,997 DEBUG [StoreFileOpener-3669efda7c21025c952e3223a054c5d8-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/family/13a696c4f20b4bb687f7300ca4a9abaf, delay=942 ms 2024-11-14T02:46:52,998 DEBUG [StoreOpener-3669efda7c21025c952e3223a054c5d8-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/family/13a696c4f20b4bb687f7300ca4a9abaf 2024-11-14T02:46:52,998 INFO [StoreOpener-3669efda7c21025c952e3223a054c5d8-1 {}] regionserver.HStore(327): Store=3669efda7c21025c952e3223a054c5d8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:52,998 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:52,999 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:53,000 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:53,000 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:53,000 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:53,004 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:53,005 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 3669efda7c21025c952e3223a054c5d8; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72617672, jitterRate=0.08208763599395752}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T02:46:53,005 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:53,006 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 3669efda7c21025c952e3223a054c5d8: Running coprocessor pre-open hook at 1731552412992Writing region info on filesystem at 1731552412993 (+1 ms)Initializing all the Stores at 1731552412993Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552412993Cleaning up temporary data from old regions at 1731552413000 (+7 ms)Running coprocessor post-open hooks at 1731552413005 (+5 ms)Region opened successfully at 1731552413006 (+1 ms) 2024-11-14T02:46:53,006 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8., pid=9, masterSystemTime=1731552412984 2024-11-14T02:46:53,009 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:53,009 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:53,009 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3669efda7c21025c952e3223a054c5d8, regionState=OPEN, openSeqNum=10, regionLocation=e5dc8b5cf5e9,42581,1731552408075 2024-11-14T02:46:53,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3669efda7c21025c952e3223a054c5d8, server=e5dc8b5cf5e9,42581,1731552408075 because future has completed 2024-11-14T02:46:53,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-14T02:46:53,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 3669efda7c21025c952e3223a054c5d8, server=e5dc8b5cf5e9,42581,1731552408075 in 181 msec 2024-11-14T02:46:53,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731552410625, region=3669efda7c21025c952e3223a054c5d8, REOPEN/MOVE in 527 msec 2024-11-14T02:46:53,041 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T02:46:53,042 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:46:53,042 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:53,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:53,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:53,047 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:53,047 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T02:46:53,047 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1058988911, stopped=false 2024-11-14T02:46:53,047 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e5dc8b5cf5e9,41647,1731552407099 2024-11-14T02:46:53,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T02:46:53,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T02:46:53,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T02:46:53,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:53,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:53,056 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T02:46:53,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:53,057 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:46:53,057 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:53,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:53,058 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:46:53,058 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e5dc8b5cf5e9,42581,1731552408075' ***** 2024-11-14T02:46:53,058 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:46:53,058 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T02:46:53,058 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:46:53,058 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e5dc8b5cf5e9,41247,1731552408278' ***** 2024-11-14T02:46:53,058 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T02:46:53,058 INFO [RS:0;e5dc8b5cf5e9:42581 {}] bucket.BucketCache(1739): Disabling cache 2024-11-14T02:46:53,058 INFO [RS:1;e5dc8b5cf5e9:41247 {}] bucket.BucketCache(1739): Disabling cache 2024-11-14T02:46:53,059 INFO [RS:0;e5dc8b5cf5e9:42581 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/bucket.persistence 2024-11-14T02:46:53,059 INFO [RS:1;e5dc8b5cf5e9:41247 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/bucket.persistence 2024-11-14T02:46:53,059 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-14T02:46:53,059 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-14T02:46:53,059 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-14T02:46:53,059 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-14T02:46:53,059 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-14T02:46:53,059 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-14T02:46:53,059 INFO [RS:1;e5dc8b5cf5e9:41247 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-14T02:46:53,060 INFO [RS:0;e5dc8b5cf5e9:42581 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-14T02:46:53,060 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-14T02:46:53,060 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-14T02:46:53,070 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] bucket.BucketCache(1488): Thread RS:1;e5dc8b5cf5e9:41247 started persisting bucket cache to file 2024-11-14T02:46:53,070 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] bucket.BucketCache(1488): Thread RS:0;e5dc8b5cf5e9:42581 started persisting bucket cache to file 2024-11-14T02:46:53,070 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T02:46:53,070 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T02:46:53,070 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T02:46:53,070 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T02:46:53,073 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-14T02:46:53,073 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-14T02:46:53,073 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731552413070 2024-11-14T02:46:53,073 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731552413070 2024-11-14T02:46:53,073 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] bucket.BucketCache(1510): Thread RS:0;e5dc8b5cf5e9:42581 finished persisting bucket cache to file, renaming 2024-11-14T02:46:53,073 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] bucket.BucketCache(1510): Thread RS:1;e5dc8b5cf5e9:41247 finished persisting bucket cache to file, renaming 2024-11-14T02:46:53,073 WARN [RS:0;e5dc8b5cf5e9:42581 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-14T02:46:53,074 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T02:46:53,074 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T02:46:53,074 INFO [RS:1;e5dc8b5cf5e9:41247 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T02:46:53,074 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T02:46:53,074 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T02:46:53,074 INFO [RS:0;e5dc8b5cf5e9:42581 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T02:46:53,074 INFO [RS:1;e5dc8b5cf5e9:41247 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T02:46:53,074 INFO [RS:0;e5dc8b5cf5e9:42581 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T02:46:53,074 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HRegionServer(959): stopping server e5dc8b5cf5e9,41247,1731552408278 2024-11-14T02:46:53,074 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T02:46:53,074 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HRegionServer(3091): Received CLOSE for 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:53,074 INFO [RS:1;e5dc8b5cf5e9:41247 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;e5dc8b5cf5e9:41247. 2024-11-14T02:46:53,074 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:53,074 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:53,075 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T02:46:53,075 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HRegionServer(959): stopping server e5dc8b5cf5e9,42581,1731552408075 2024-11-14T02:46:53,075 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T02:46:53,075 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T02:46:53,075 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T02:46:53,075 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T02:46:53,075 INFO [RS:0;e5dc8b5cf5e9:42581 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e5dc8b5cf5e9:42581. 2024-11-14T02:46:53,075 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:53,075 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:53,075 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T02:46:53,075 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-14T02:46:53,075 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T02:46:53,075 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T02:46:53,075 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HRegionServer(1325): Online Regions={3669efda7c21025c952e3223a054c5d8=testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8.} 2024-11-14T02:46:53,075 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-14T02:46:53,075 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3669efda7c21025c952e3223a054c5d8, disabling compactions & flushes 2024-11-14T02:46:53,075 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:53,075 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T02:46:53,075 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HRegionServer(1351): Waiting on 3669efda7c21025c952e3223a054c5d8 2024-11-14T02:46:53,075 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T02:46:53,076 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:53,076 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. after waiting 0 ms 2024-11-14T02:46:53,076 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T02:46:53,076 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:53,076 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T02:46:53,076 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T02:46:53,076 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T02:46:53,076 DEBUG [StoreFileCloser-3669efda7c21025c952e3223a054c5d8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/family/13a696c4f20b4bb687f7300ca4a9abaf 2024-11-14T02:46:53,076 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-14T02:46:53,080 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnRegionMove1731552410625/3669efda7c21025c952e3223a054c5d8/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-14T02:46:53,081 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:53,081 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3669efda7c21025c952e3223a054c5d8: Waiting for close lock at 1731552413075Running coprocessor pre-close hooks at 1731552413075Disabling compacts and flushes for region at 1731552413075Disabling writes for close at 1731552413076 (+1 ms)Writing region close event to WAL at 1731552413077 (+1 ms)Running coprocessor post-close hooks at 1731552413081 (+4 ms)Closed at 1731552413081 2024-11-14T02:46:53,081 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8. 2024-11-14T02:46:53,085 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 861491b9f5714236961f89f9075db688 2024-11-14T02:46:53,104 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/info/861491b9f5714236961f89f9075db688 is 205, key is testBlockEvictionOnRegionMove1731552410625,,1731552410646.3669efda7c21025c952e3223a054c5d8./info:regioninfo/1731552413009/Put/seqid=0 2024-11-14T02:46:53,105 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/info/861491b9f5714236961f89f9075db688 2024-11-14T02:46:53,107 INFO [regionserver/e5dc8b5cf5e9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T02:46:53,111 INFO [regionserver/e5dc8b5cf5e9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T02:46:53,113 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3491f69f6ef940cd9900e80fb2ea965a 2024-11-14T02:46:53,126 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/ns/3491f69f6ef940cd9900e80fb2ea965a is 43, key is default/ns:d/1731552410399/Put/seqid=0 2024-11-14T02:46:53,127 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/ns/3491f69f6ef940cd9900e80fb2ea965a 2024-11-14T02:46:53,135 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7e8435b4c854400da22f2316be2c13b6 2024-11-14T02:46:53,146 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/table/7e8435b4c854400da22f2316be2c13b6 is 78, key is testBlockEvictionOnRegionMove1731552410625/table:state/1731552411064/Put/seqid=0 2024-11-14T02:46:53,147 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/table/7e8435b4c854400da22f2316be2c13b6 2024-11-14T02:46:53,149 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/info/861491b9f5714236961f89f9075db688 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/info/861491b9f5714236961f89f9075db688 2024-11-14T02:46:53,150 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/info/861491b9f5714236961f89f9075db688, delay=917 ms 2024-11-14T02:46:53,152 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/info/861491b9f5714236961f89f9075db688, entries=15, sequenceid=15, filesize=7.6 K 2024-11-14T02:46:53,152 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/ns/3491f69f6ef940cd9900e80fb2ea965a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/ns/3491f69f6ef940cd9900e80fb2ea965a 2024-11-14T02:46:53,153 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/ns/3491f69f6ef940cd9900e80fb2ea965a, delay=950 ms 2024-11-14T02:46:53,154 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/ns/3491f69f6ef940cd9900e80fb2ea965a, entries=2, sequenceid=15, filesize=5.0 K 2024-11-14T02:46:53,155 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/table/7e8435b4c854400da22f2316be2c13b6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/table/7e8435b4c854400da22f2316be2c13b6 2024-11-14T02:46:53,156 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/table/7e8435b4c854400da22f2316be2c13b6, delay=973 ms 2024-11-14T02:46:53,157 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/table/7e8435b4c854400da22f2316be2c13b6, entries=2, sequenceid=15, filesize=5.3 K 2024-11-14T02:46:53,157 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=15, compaction requested=false 2024-11-14T02:46:53,157 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T02:46:53,159 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/info/861491b9f5714236961f89f9075db688 2024-11-14T02:46:53,159 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/ns/3491f69f6ef940cd9900e80fb2ea965a 2024-11-14T02:46:53,160 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/table/7e8435b4c854400da22f2316be2c13b6 2024-11-14T02:46:53,164 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-14T02:46:53,164 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T02:46:53,164 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T02:46:53,165 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731552413075Running coprocessor pre-close hooks at 1731552413075Disabling compacts and flushes for region at 1731552413075Disabling writes for close at 1731552413076 (+1 ms)Obtaining lock to block concurrent updates at 1731552413076Preparing flush snapshotting stores in 1588230740 at 1731552413076Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731552413077 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731552413077Flushing 1588230740/info: creating writer at 1731552413077Flushing 1588230740/info: appending metadata at 1731552413103 (+26 ms)Flushing 1588230740/info: closing flushed file at 1731552413103Flushing 1588230740/ns: creating writer at 1731552413108 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731552413125 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731552413125Flushing 1588230740/table: creating writer at 1731552413129 (+4 ms)Flushing 1588230740/table: appending metadata at 1731552413146 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731552413146Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1266b279: reopening flushed file at 1731552413149 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5eec8dd: reopening flushed file at 1731552413152 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17f65851: reopening flushed file at 1731552413154 (+2 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=15, compaction requested=false at 1731552413157 (+3 ms)Writing region close event to WAL at 1731552413160 (+3 ms)Running coprocessor post-close hooks at 1731552413164 (+4 ms)Closed at 1731552413164 2024-11-14T02:46:53,165 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T02:46:53,276 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HRegionServer(976): stopping server e5dc8b5cf5e9,42581,1731552408075; all regions closed. 2024-11-14T02:46:53,276 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HRegionServer(976): stopping server e5dc8b5cf5e9,41247,1731552408278; all regions closed. 2024-11-14T02:46:53,286 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/oldWALs 2024-11-14T02:46:53,286 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/oldWALs 2024-11-14T02:46:53,287 INFO [RS:1;e5dc8b5cf5e9:41247 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e5dc8b5cf5e9%2C41247%2C1731552408278.meta:.meta(num 1731552410249) 2024-11-14T02:46:53,287 INFO [RS:0;e5dc8b5cf5e9:42581 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e5dc8b5cf5e9%2C42581%2C1731552408075:(num 1731552409766) 2024-11-14T02:46:53,287 DEBUG [RS:0;e5dc8b5cf5e9:42581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:53,287 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T02:46:53,288 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T02:46:53,288 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.ChoreService(370): Chore service for: regionserver/e5dc8b5cf5e9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T02:46:53,288 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T02:46:53,288 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T02:46:53,288 INFO [regionserver/e5dc8b5cf5e9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T02:46:53,288 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T02:46:53,288 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T02:46:53,289 INFO [RS:0;e5dc8b5cf5e9:42581 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42581 2024-11-14T02:46:53,290 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/oldWALs 2024-11-14T02:46:53,290 INFO [RS:1;e5dc8b5cf5e9:41247 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e5dc8b5cf5e9%2C41247%2C1731552408278:(num 1731552409766) 2024-11-14T02:46:53,290 DEBUG [RS:1;e5dc8b5cf5e9:41247 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:53,290 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T02:46:53,290 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T02:46:53,290 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.ChoreService(370): Chore service for: regionserver/e5dc8b5cf5e9:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T02:46:53,290 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T02:46:53,290 INFO [regionserver/e5dc8b5cf5e9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T02:46:53,291 INFO [RS:1;e5dc8b5cf5e9:41247 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41247 2024-11-14T02:46:53,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e5dc8b5cf5e9,42581,1731552408075 2024-11-14T02:46:53,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T02:46:53,306 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T02:46:53,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e5dc8b5cf5e9,41247,1731552408278 2024-11-14T02:46:53,306 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T02:46:53,320 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e5dc8b5cf5e9,41247,1731552408278] 2024-11-14T02:46:53,330 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e5dc8b5cf5e9,41247,1731552408278 already deleted, retry=false 2024-11-14T02:46:53,330 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e5dc8b5cf5e9,41247,1731552408278 expired; onlineServers=1 2024-11-14T02:46:53,330 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e5dc8b5cf5e9,42581,1731552408075] 2024-11-14T02:46:53,340 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e5dc8b5cf5e9,42581,1731552408075 already deleted, retry=false 2024-11-14T02:46:53,340 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e5dc8b5cf5e9,42581,1731552408075 expired; onlineServers=0 2024-11-14T02:46:53,340 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e5dc8b5cf5e9,41647,1731552407099' ***** 2024-11-14T02:46:53,340 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T02:46:53,341 INFO [M:0;e5dc8b5cf5e9:41647 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T02:46:53,341 INFO [M:0;e5dc8b5cf5e9:41647 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T02:46:53,341 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T02:46:53,341 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T02:46:53,341 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T02:46:53,341 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.small.0-1731552409456 {}] cleaner.HFileCleaner(306): Exit Thread[master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.small.0-1731552409456,5,FailOnTimeoutGroup] 2024-11-14T02:46:53,341 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.large.0-1731552409455 {}] cleaner.HFileCleaner(306): Exit Thread[master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.large.0-1731552409455,5,FailOnTimeoutGroup] 2024-11-14T02:46:53,341 INFO [M:0;e5dc8b5cf5e9:41647 {}] hbase.ChoreService(370): Chore service for: master/e5dc8b5cf5e9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T02:46:53,341 INFO [M:0;e5dc8b5cf5e9:41647 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T02:46:53,342 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] master.HMaster(1795): Stopping service threads 2024-11-14T02:46:53,342 INFO [M:0;e5dc8b5cf5e9:41647 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T02:46:53,342 INFO [M:0;e5dc8b5cf5e9:41647 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T02:46:53,342 INFO [M:0;e5dc8b5cf5e9:41647 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T02:46:53,343 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T02:46:53,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T02:46:53,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:53,351 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] zookeeper.ZKUtil(347): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T02:46:53,351 WARN [M:0;e5dc8b5cf5e9:41647 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T02:46:53,352 INFO [M:0;e5dc8b5cf5e9:41647 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/.lastflushedseqids 2024-11-14T02:46:53,359 INFO [M:0;e5dc8b5cf5e9:41647 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T02:46:53,359 INFO [M:0;e5dc8b5cf5e9:41647 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T02:46:53,359 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T02:46:53,359 INFO [M:0;e5dc8b5cf5e9:41647 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:53,359 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:53,359 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T02:46:53,359 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:53,359 INFO [M:0;e5dc8b5cf5e9:41647 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-11-14T02:46:53,369 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b50fe5b71f6e44289bfa1eb91936a4a5 2024-11-14T02:46:53,382 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b50fe5b71f6e44289bfa1eb91936a4a5 is 82, key is hbase:meta,,1/info:regioninfo/1731552410317/Put/seqid=0 2024-11-14T02:46:53,383 INFO [M:0;e5dc8b5cf5e9:41647 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b50fe5b71f6e44289bfa1eb91936a4a5 2024-11-14T02:46:53,390 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e7274ba90a664cd3ad0dba20938fca50 2024-11-14T02:46:53,403 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e7274ba90a664cd3ad0dba20938fca50 is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731552411071/Put/seqid=0 2024-11-14T02:46:53,404 INFO [M:0;e5dc8b5cf5e9:41647 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e7274ba90a664cd3ad0dba20938fca50 2024-11-14T02:46:53,411 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 79d5305bc5b0459a9cac7c4bc6e9be05 2024-11-14T02:46:53,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:53,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:53,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41247-0x101373ce89c0002, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:53,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42581-0x101373ce89c0001, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:53,420 INFO [RS:1;e5dc8b5cf5e9:41247 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T02:46:53,420 INFO [RS:0;e5dc8b5cf5e9:42581 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T02:46:53,420 INFO [RS:1;e5dc8b5cf5e9:41247 {}] regionserver.HRegionServer(1031): Exiting; stopping=e5dc8b5cf5e9,41247,1731552408278; zookeeper connection closed. 2024-11-14T02:46:53,420 INFO [RS:0;e5dc8b5cf5e9:42581 {}] regionserver.HRegionServer(1031): Exiting; stopping=e5dc8b5cf5e9,42581,1731552408075; zookeeper connection closed. 2024-11-14T02:46:53,421 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1505493a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1505493a 2024-11-14T02:46:53,421 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3dcc8087 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3dcc8087 2024-11-14T02:46:53,421 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-14T02:46:53,424 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/79d5305bc5b0459a9cac7c4bc6e9be05 is 69, key is e5dc8b5cf5e9,41247,1731552408278/rs:state/1731552409482/Put/seqid=0 2024-11-14T02:46:53,425 INFO [M:0;e5dc8b5cf5e9:41647 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/79d5305bc5b0459a9cac7c4bc6e9be05 2024-11-14T02:46:53,427 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b50fe5b71f6e44289bfa1eb91936a4a5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b50fe5b71f6e44289bfa1eb91936a4a5 2024-11-14T02:46:53,428 INFO [M:0;e5dc8b5cf5e9:41647 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b50fe5b71f6e44289bfa1eb91936a4a5, entries=8, sequenceid=84, filesize=5.5 K 2024-11-14T02:46:53,429 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e7274ba90a664cd3ad0dba20938fca50 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e7274ba90a664cd3ad0dba20938fca50 2024-11-14T02:46:53,430 INFO [M:0;e5dc8b5cf5e9:41647 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e7274ba90a664cd3ad0dba20938fca50, entries=9, sequenceid=84, filesize=6.5 K 2024-11-14T02:46:53,431 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/79d5305bc5b0459a9cac7c4bc6e9be05 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/79d5305bc5b0459a9cac7c4bc6e9be05 2024-11-14T02:46:53,432 INFO [M:0;e5dc8b5cf5e9:41647 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/79d5305bc5b0459a9cac7c4bc6e9be05, entries=2, sequenceid=84, filesize=5.1 K 2024-11-14T02:46:53,433 INFO [M:0;e5dc8b5cf5e9:41647 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 74ms, sequenceid=84, compaction requested=false 2024-11-14T02:46:53,435 INFO [M:0;e5dc8b5cf5e9:41647 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:53,435 DEBUG [M:0;e5dc8b5cf5e9:41647 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731552413359Disabling compacts and flushes for region at 1731552413359Disabling writes for close at 1731552413359Obtaining lock to block concurrent updates at 1731552413360 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731552413360Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1731552413360Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731552413360Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731552413360Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731552413381 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731552413381Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731552413384 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731552413402 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731552413403 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731552413406 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731552413424 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731552413424Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@699f71b4: reopening flushed file at 1731552413426 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@173598f9: reopening flushed file at 1731552413429 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1baad2f5: reopening flushed file at 1731552413430 (+1 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 74ms, sequenceid=84, compaction requested=false at 1731552413433 (+3 ms)Writing region close event to WAL at 1731552413435 (+2 ms)Closed at 1731552413435 2024-11-14T02:46:53,436 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T02:46:53,436 INFO [M:0;e5dc8b5cf5e9:41647 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T02:46:53,437 INFO [M:0;e5dc8b5cf5e9:41647 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41647 2024-11-14T02:46:53,437 INFO [M:0;e5dc8b5cf5e9:41647 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T02:46:53,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:53,546 INFO [M:0;e5dc8b5cf5e9:41647 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T02:46:53,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41647-0x101373ce89c0000, quorum=127.0.0.1:54067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:53,553 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T02:46:53,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-14T02:46:53,575 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:53,582 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/e5dc8b5cf5e9:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/e5dc8b5cf5e9:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/e5dc8b5cf5e9:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/e5dc8b5cf5e9:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1731552412997 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731552413153 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731552413150 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731552411455 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=447 (was 487), ProcessCount=11 (was 11), AvailableMemoryMB=11068 (was 11647) 2024-11-14T02:46:53,587 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=447, ProcessCount=11, AvailableMemoryMB=11068 2024-11-14T02:46:53,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/hadoop-log-dir so I do NOT create it in target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf 2024-11-14T02:46:53,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/hadoop-tmp-dir so I do NOT create it in target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf 2024-11-14T02:46:53,588 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:53,590 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/cluster_bb07fae3-6d57-5107-a441-8a0cac83cca1, deleteOnExit=true 2024-11-14T02:46:53,593 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/cluster_bb07fae3-6d57-5107-a441-8a0cac83cca1/zookeeper_0, clientPort=51661, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/cluster_bb07fae3-6d57-5107-a441-8a0cac83cca1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/cluster_bb07fae3-6d57-5107-a441-8a0cac83cca1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T02:46:53,594 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51661 2024-11-14T02:46:53,605 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414 with version=8 2024-11-14T02:46:53,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/hbase-staging 2024-11-14T02:46:53,607 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e5dc8b5cf5e9:0 server-side Connection retries=45 2024-11-14T02:46:53,607 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:53,608 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:53,608 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T02:46:53,608 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:53,608 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T02:46:53,608 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T02:46:53,608 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T02:46:53,609 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43215 2024-11-14T02:46:53,611 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43215 connecting to ZooKeeper ensemble=127.0.0.1:51661 2024-11-14T02:46:53,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:432150x0, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T02:46:53,656 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43215-0x101373d03430000 connected 2024-11-14T02:46:53,763 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:53,763 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:53,767 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:46:53,768 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414, hbase.cluster.distributed=false 2024-11-14T02:46:53,770 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T02:46:53,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43215 2024-11-14T02:46:53,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43215 2024-11-14T02:46:53,773 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43215 2024-11-14T02:46:53,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43215 2024-11-14T02:46:53,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43215 2024-11-14T02:46:53,798 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e5dc8b5cf5e9:0 server-side Connection retries=45 2024-11-14T02:46:53,798 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:53,798 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:53,798 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T02:46:53,798 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:53,799 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T02:46:53,799 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T02:46:53,799 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T02:46:53,800 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35911 2024-11-14T02:46:53,802 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35911 connecting to ZooKeeper ensemble=127.0.0.1:51661 2024-11-14T02:46:53,802 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:53,802 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:53,815 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359110x0, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T02:46:53,815 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35911-0x101373d03430001 connected 2024-11-14T02:46:53,815 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:46:53,816 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T02:46:53,817 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/bucket.cache 2024-11-14T02:46:53,817 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-14T02:46:53,817 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-14T02:46:53,817 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-14T02:46:53,817 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-14T02:46:53,818 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-14T02:46:53,818 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-14T02:46:53,818 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-14T02:46:53,818 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/bucket.persistence is complete. 2024-11-14T02:46:53,819 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T02:46:53,820 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T02:46:53,821 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T02:46:53,825 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35911 2024-11-14T02:46:53,825 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35911 2024-11-14T02:46:53,826 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35911 2024-11-14T02:46:53,827 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35911 2024-11-14T02:46:53,828 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35911 2024-11-14T02:46:53,845 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e5dc8b5cf5e9:0 server-side Connection retries=45 2024-11-14T02:46:53,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:53,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:53,845 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T02:46:53,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:53,845 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T02:46:53,845 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T02:46:53,845 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T02:46:53,846 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33277 2024-11-14T02:46:53,848 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33277 connecting to ZooKeeper ensemble=127.0.0.1:51661 2024-11-14T02:46:53,848 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:53,848 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:53,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:332770x0, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T02:46:53,857 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:332770x0, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:46:53,857 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33277-0x101373d03430002 connected 2024-11-14T02:46:53,858 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T02:46:53,859 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/bucket.cache 2024-11-14T02:46:53,859 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-14T02:46:53,859 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-14T02:46:53,859 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-14T02:46:53,860 INFO [Thread-121 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-14T02:46:53,860 WARN [Thread-121 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-14T02:46:53,860 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-14T02:46:53,860 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-14T02:46:53,860 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T02:46:53,860 INFO [Thread-121 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/bucket.persistence is complete. 2024-11-14T02:46:53,862 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T02:46:53,863 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T02:46:53,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33277 2024-11-14T02:46:53,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33277 2024-11-14T02:46:53,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33277 2024-11-14T02:46:53,866 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33277 2024-11-14T02:46:53,866 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33277 2024-11-14T02:46:53,879 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e5dc8b5cf5e9:43215 2024-11-14T02:46:53,881 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e5dc8b5cf5e9,43215,1731552413607 2024-11-14T02:46:53,888 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:46:53,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:46:53,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:46:53,889 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e5dc8b5cf5e9,43215,1731552413607 2024-11-14T02:46:53,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T02:46:53,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:53,899 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T02:46:53,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:53,899 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:53,900 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T02:46:53,900 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e5dc8b5cf5e9,43215,1731552413607 from backup master directory 2024-11-14T02:46:53,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:46:53,909 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:46:53,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e5dc8b5cf5e9,43215,1731552413607 2024-11-14T02:46:53,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T02:46:53,910 WARN [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T02:46:53,910 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e5dc8b5cf5e9,43215,1731552413607 2024-11-14T02:46:53,912 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/hbase.id] with ID: 83dfadfb-2547-4d23-aa00-fb6cbbf7b8fb 2024-11-14T02:46:53,912 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/.tmp/hbase.id 2024-11-14T02:46:53,916 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/hbase.id] 2024-11-14T02:46:53,952 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:53,952 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T02:46:53,955 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-14T02:46:53,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:53,969 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:53,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:53,976 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:46:53,977 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T02:46:53,978 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T02:46:54,001 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store 2024-11-14T02:46:54,008 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:54,008 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T02:46:54,008 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:54,009 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:54,009 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T02:46:54,009 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:54,009 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:54,009 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731552414008Disabling compacts and flushes for region at 1731552414008Disabling writes for close at 1731552414009 (+1 ms)Writing region close event to WAL at 1731552414009Closed at 1731552414009 2024-11-14T02:46:54,009 WARN [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/.initializing 2024-11-14T02:46:54,009 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/WALs/e5dc8b5cf5e9,43215,1731552413607 2024-11-14T02:46:54,023 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T02:46:54,026 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e5dc8b5cf5e9%2C43215%2C1731552413607, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/WALs/e5dc8b5cf5e9,43215,1731552413607, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/oldWALs, maxLogs=10 2024-11-14T02:46:54,045 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/WALs/e5dc8b5cf5e9,43215,1731552413607/e5dc8b5cf5e9%2C43215%2C1731552413607.1731552414026 2024-11-14T02:46:54,045 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T02:46:54,045 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:46:54,046 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:54,046 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:54,046 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:54,047 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:54,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T02:46:54,052 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:54,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:54,053 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:54,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T02:46:54,056 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:54,057 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:54,057 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:54,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T02:46:54,061 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:54,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:54,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:54,066 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T02:46:54,066 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:54,066 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:54,066 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:54,067 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:54,067 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:54,067 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:54,067 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:54,067 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T02:46:54,068 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T02:46:54,081 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:46:54,081 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64423571, jitterRate=-0.040013983845710754}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T02:46:54,082 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731552414046Initializing all the Stores at 1731552414046Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552414047 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552414047Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552414047Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552414047Cleaning up temporary data from old regions at 1731552414067 (+20 ms)Region opened successfully at 1731552414082 (+15 ms) 2024-11-14T02:46:54,083 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T02:46:54,087 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328acda3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e5dc8b5cf5e9/172.17.0.2:0 2024-11-14T02:46:54,088 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T02:46:54,089 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T02:46:54,089 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T02:46:54,089 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T02:46:54,089 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T02:46:54,090 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-14T02:46:54,090 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T02:46:54,099 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T02:46:54,100 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T02:46:54,109 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T02:46:54,110 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T02:46:54,111 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T02:46:54,120 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T02:46:54,120 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T02:46:54,121 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T02:46:54,130 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T02:46:54,132 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T02:46:54,141 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T02:46:54,144 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T02:46:54,151 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T02:46:54,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T02:46:54,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T02:46:54,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:54,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:54,162 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T02:46:54,162 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:54,162 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e5dc8b5cf5e9,43215,1731552413607, sessionid=0x101373d03430000, setting cluster-up flag (Was=false) 2024-11-14T02:46:54,183 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:54,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:54,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:54,214 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T02:46:54,216 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e5dc8b5cf5e9,43215,1731552413607 2024-11-14T02:46:54,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:54,236 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:54,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:54,278 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T02:46:54,279 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e5dc8b5cf5e9,43215,1731552413607 2024-11-14T02:46:54,285 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T02:46:54,288 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T02:46:54,289 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T02:46:54,289 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T02:46:54,289 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e5dc8b5cf5e9,43215,1731552413607 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T02:46:54,292 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e5dc8b5cf5e9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T02:46:54,292 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e5dc8b5cf5e9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T02:46:54,293 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e5dc8b5cf5e9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T02:46:54,293 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e5dc8b5cf5e9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T02:46:54,293 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e5dc8b5cf5e9:0, corePoolSize=10, maxPoolSize=10 2024-11-14T02:46:54,293 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,293 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e5dc8b5cf5e9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T02:46:54,293 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,299 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T02:46:54,300 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T02:46:54,302 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:54,302 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T02:46:54,304 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731552444304 2024-11-14T02:46:54,305 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T02:46:54,305 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T02:46:54,305 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T02:46:54,305 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T02:46:54,305 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T02:46:54,305 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T02:46:54,306 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,309 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T02:46:54,309 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T02:46:54,309 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T02:46:54,310 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T02:46:54,311 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T02:46:54,311 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.large.0-1731552414311,5,FailOnTimeoutGroup] 2024-11-14T02:46:54,311 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.small.0-1731552414311,5,FailOnTimeoutGroup] 2024-11-14T02:46:54,311 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,311 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T02:46:54,311 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,311 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,321 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T02:46:54,321 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414 2024-11-14T02:46:54,330 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:54,331 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-14T02:46:54,334 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T02:46:54,334 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:54,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:54,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-14T02:46:54,337 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T02:46:54,338 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:54,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:54,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-14T02:46:54,341 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T02:46:54,341 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:54,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:54,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-14T02:46:54,344 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T02:46:54,344 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:54,344 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:54,345 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T02:46:54,345 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740 2024-11-14T02:46:54,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740 2024-11-14T02:46:54,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T02:46:54,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T02:46:54,346 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T02:46:54,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T02:46:54,353 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:46:54,353 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64316835, jitterRate=-0.04160447418689728}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T02:46:54,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731552414330Initializing all the Stores at 1731552414330Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552414330Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552414331 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552414331Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552414331Cleaning up temporary data from old regions at 1731552414346 (+15 ms)Region opened successfully at 1731552414354 (+8 ms) 2024-11-14T02:46:54,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T02:46:54,355 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T02:46:54,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T02:46:54,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T02:46:54,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T02:46:54,355 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T02:46:54,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731552414354Disabling compacts and flushes for region at 1731552414354Disabling writes for close at 1731552414355 (+1 ms)Writing region close event to WAL at 1731552414355Closed at 1731552414355 2024-11-14T02:46:54,356 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T02:46:54,356 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T02:46:54,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T02:46:54,358 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T02:46:54,358 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T02:46:54,369 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer(746): ClusterId : 83dfadfb-2547-4d23-aa00-fb6cbbf7b8fb 2024-11-14T02:46:54,369 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HRegionServer(746): ClusterId : 83dfadfb-2547-4d23-aa00-fb6cbbf7b8fb 2024-11-14T02:46:54,369 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T02:46:54,369 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T02:46:54,381 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T02:46:54,381 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T02:46:54,381 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T02:46:54,381 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T02:46:54,394 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T02:46:54,394 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T02:46:54,395 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a769239, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e5dc8b5cf5e9/172.17.0.2:0 2024-11-14T02:46:54,395 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62b913, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e5dc8b5cf5e9/172.17.0.2:0 2024-11-14T02:46:54,408 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;e5dc8b5cf5e9:33277 2024-11-14T02:46:54,408 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T02:46:54,408 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T02:46:54,408 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T02:46:54,409 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HRegionServer(2659): reportForDuty to master=e5dc8b5cf5e9,43215,1731552413607 with port=33277, startcode=1731552413844 2024-11-14T02:46:54,409 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T02:46:54,412 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e5dc8b5cf5e9:35911 2024-11-14T02:46:54,413 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T02:46:54,413 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T02:46:54,413 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T02:46:54,414 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54317, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T02:46:54,415 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43215 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e5dc8b5cf5e9,33277,1731552413844 2024-11-14T02:46:54,415 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43215 {}] master.ServerManager(517): Registering regionserver=e5dc8b5cf5e9,33277,1731552413844 2024-11-14T02:46:54,417 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414 2024-11-14T02:46:54,417 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-14T02:46:54,417 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer(2659): reportForDuty to master=e5dc8b5cf5e9,43215,1731552413607 with port=35911, startcode=1731552413798 2024-11-14T02:46:54,417 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T02:46:54,417 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T02:46:54,421 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59973, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T02:46:54,421 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43215 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e5dc8b5cf5e9,35911,1731552413798 2024-11-14T02:46:54,422 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43215 {}] master.ServerManager(517): Registering regionserver=e5dc8b5cf5e9,35911,1731552413798 2024-11-14T02:46:54,423 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414 2024-11-14T02:46:54,423 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-14T02:46:54,423 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T02:46:54,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T02:46:54,502 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] zookeeper.ZKUtil(111): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e5dc8b5cf5e9,33277,1731552413844 2024-11-14T02:46:54,502 WARN [RS:1;e5dc8b5cf5e9:33277 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T02:46:54,502 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e5dc8b5cf5e9,33277,1731552413844] 2024-11-14T02:46:54,502 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e5dc8b5cf5e9,35911,1731552413798] 2024-11-14T02:46:54,502 INFO [RS:1;e5dc8b5cf5e9:33277 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T02:46:54,502 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,33277,1731552413844 2024-11-14T02:46:54,502 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] zookeeper.ZKUtil(111): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e5dc8b5cf5e9,35911,1731552413798 2024-11-14T02:46:54,502 WARN [RS:0;e5dc8b5cf5e9:35911 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T02:46:54,502 INFO [RS:0;e5dc8b5cf5e9:35911 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T02:46:54,502 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,35911,1731552413798 2024-11-14T02:46:54,509 WARN [e5dc8b5cf5e9:43215 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-14T02:46:54,511 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T02:46:54,516 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T02:46:54,517 INFO [RS:0;e5dc8b5cf5e9:35911 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T02:46:54,517 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,519 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T02:46:54,524 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T02:46:54,525 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T02:46:54,525 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,526 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,526 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,526 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,526 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,526 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,526 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e5dc8b5cf5e9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T02:46:54,526 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,526 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,526 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,526 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T02:46:54,526 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,527 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,527 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,527 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e5dc8b5cf5e9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T02:46:54,527 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T02:46:54,527 INFO [RS:1;e5dc8b5cf5e9:33277 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T02:46:54,527 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,535 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T02:46:54,536 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T02:46:54,536 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,536 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,536 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,536 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,536 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,536 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,536 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,536 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,536 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,536 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,536 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,537 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,35911,1731552413798-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T02:46:54,537 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e5dc8b5cf5e9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T02:46:54,537 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,537 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,537 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,537 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,537 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,537 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:54,537 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e5dc8b5cf5e9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T02:46:54,537 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T02:46:54,551 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,551 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,551 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,551 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,551 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,551 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,33277,1731552413844-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T02:46:54,560 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T02:46:54,560 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,35911,1731552413798-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,561 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,561 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.Replication(171): e5dc8b5cf5e9,35911,1731552413798 started 2024-11-14T02:46:54,568 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T02:46:54,568 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,33277,1731552413844-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,569 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,569 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.Replication(171): e5dc8b5cf5e9,33277,1731552413844 started 2024-11-14T02:46:54,582 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,583 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer(1482): Serving as e5dc8b5cf5e9,35911,1731552413798, RpcServer on e5dc8b5cf5e9/172.17.0.2:35911, sessionid=0x101373d03430001 2024-11-14T02:46:54,583 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T02:46:54,583 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e5dc8b5cf5e9,35911,1731552413798 2024-11-14T02:46:54,583 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5dc8b5cf5e9,35911,1731552413798' 2024-11-14T02:46:54,583 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T02:46:54,584 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T02:46:54,585 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T02:46:54,585 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T02:46:54,585 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e5dc8b5cf5e9,35911,1731552413798 2024-11-14T02:46:54,585 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5dc8b5cf5e9,35911,1731552413798' 2024-11-14T02:46:54,585 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T02:46:54,586 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T02:46:54,586 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T02:46:54,586 INFO [RS:0;e5dc8b5cf5e9:35911 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T02:46:54,586 INFO [RS:0;e5dc8b5cf5e9:35911 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T02:46:54,588 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:54,588 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HRegionServer(1482): Serving as e5dc8b5cf5e9,33277,1731552413844, RpcServer on e5dc8b5cf5e9/172.17.0.2:33277, sessionid=0x101373d03430002 2024-11-14T02:46:54,589 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T02:46:54,589 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e5dc8b5cf5e9,33277,1731552413844 2024-11-14T02:46:54,589 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5dc8b5cf5e9,33277,1731552413844' 2024-11-14T02:46:54,589 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T02:46:54,590 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T02:46:54,590 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T02:46:54,590 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T02:46:54,590 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e5dc8b5cf5e9,33277,1731552413844 2024-11-14T02:46:54,590 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5dc8b5cf5e9,33277,1731552413844' 2024-11-14T02:46:54,590 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T02:46:54,591 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T02:46:54,592 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T02:46:54,592 INFO [RS:1;e5dc8b5cf5e9:33277 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T02:46:54,592 INFO [RS:1;e5dc8b5cf5e9:33277 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T02:46:54,687 INFO [RS:0;e5dc8b5cf5e9:35911 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T02:46:54,688 INFO [RS:0;e5dc8b5cf5e9:35911 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e5dc8b5cf5e9%2C35911%2C1731552413798, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,35911,1731552413798, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/oldWALs, maxLogs=55 2024-11-14T02:46:54,693 INFO [RS:1;e5dc8b5cf5e9:33277 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T02:46:54,693 INFO [RS:1;e5dc8b5cf5e9:33277 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e5dc8b5cf5e9%2C33277%2C1731552413844, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,33277,1731552413844, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/oldWALs, maxLogs=55 2024-11-14T02:46:54,707 INFO [RS:0;e5dc8b5cf5e9:35911 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,35911,1731552413798/e5dc8b5cf5e9%2C35911%2C1731552413798.1731552414690 2024-11-14T02:46:54,707 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T02:46:54,711 INFO [RS:1;e5dc8b5cf5e9:33277 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,33277,1731552413844/e5dc8b5cf5e9%2C33277%2C1731552413844.1731552414694 2024-11-14T02:46:54,712 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T02:46:54,760 DEBUG [e5dc8b5cf5e9:43215 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-14T02:46:54,760 DEBUG [e5dc8b5cf5e9:43215 {}] balancer.BalancerClusterState(204): Hosts are {e5dc8b5cf5e9=0} racks are {/default-rack=0} 2024-11-14T02:46:54,762 DEBUG [e5dc8b5cf5e9:43215 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T02:46:54,762 DEBUG [e5dc8b5cf5e9:43215 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T02:46:54,762 DEBUG [e5dc8b5cf5e9:43215 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T02:46:54,762 DEBUG [e5dc8b5cf5e9:43215 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T02:46:54,762 INFO [e5dc8b5cf5e9:43215 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T02:46:54,762 INFO [e5dc8b5cf5e9:43215 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T02:46:54,762 DEBUG [e5dc8b5cf5e9:43215 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T02:46:54,763 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e5dc8b5cf5e9,35911,1731552413798 2024-11-14T02:46:54,765 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e5dc8b5cf5e9,35911,1731552413798, state=OPENING 2024-11-14T02:46:54,779 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T02:46:54,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:54,790 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:54,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:54,791 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T02:46:54,791 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:46:54,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e5dc8b5cf5e9,35911,1731552413798}] 2024-11-14T02:46:54,791 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:46:54,792 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:46:54,945 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T02:46:54,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47989, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T02:46:54,952 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T02:46:54,952 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T02:46:54,953 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T02:46:54,954 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e5dc8b5cf5e9%2C35911%2C1731552413798.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,35911,1731552413798, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/oldWALs, maxLogs=55 2024-11-14T02:46:54,972 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,35911,1731552413798/e5dc8b5cf5e9%2C35911%2C1731552413798.meta.1731552414955.meta 2024-11-14T02:46:54,973 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T02:46:54,973 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:46:54,973 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T02:46:54,973 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T02:46:54,974 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T02:46:54,974 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T02:46:54,974 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:54,974 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T02:46:54,974 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T02:46:54,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-14T02:46:54,976 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T02:46:54,976 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:54,976 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:54,976 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-14T02:46:54,976 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T02:46:54,976 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:54,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:54,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-14T02:46:54,977 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T02:46:54,977 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:54,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:54,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-14T02:46:54,978 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T02:46:54,978 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:54,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T02:46:54,978 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T02:46:54,979 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740 2024-11-14T02:46:54,979 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740 2024-11-14T02:46:54,979 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T02:46:54,979 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T02:46:54,980 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T02:46:54,980 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T02:46:54,981 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63167101, jitterRate=-0.05873684585094452}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T02:46:54,981 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T02:46:54,981 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731552414974Writing region info on filesystem at 1731552414974Initializing all the Stores at 1731552414975 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552414975Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552414975Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552414975Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731552414975Cleaning up temporary data from old regions at 1731552414979 (+4 ms)Running coprocessor post-open hooks at 1731552414981 (+2 ms)Region opened successfully at 1731552414981 2024-11-14T02:46:54,982 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731552414945 2024-11-14T02:46:54,984 DEBUG [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T02:46:54,984 INFO [RS_OPEN_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T02:46:54,984 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e5dc8b5cf5e9,35911,1731552413798 2024-11-14T02:46:54,985 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e5dc8b5cf5e9,35911,1731552413798, state=OPEN 2024-11-14T02:46:54,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T02:46:54,993 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T02:46:54,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T02:46:54,993 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e5dc8b5cf5e9,35911,1731552413798 2024-11-14T02:46:54,993 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:46:54,993 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:46:54,993 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T02:46:54,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T02:46:54,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e5dc8b5cf5e9,35911,1731552413798 in 202 msec 2024-11-14T02:46:54,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T02:46:54,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 639 msec 2024-11-14T02:46:54,999 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T02:46:54,999 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T02:46:55,003 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:46:55,003 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,35911,1731552413798, seqNum=-1] 2024-11-14T02:46:55,003 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:46:55,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56829, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:46:55,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 725 msec 2024-11-14T02:46:55,011 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731552415011, completionTime=-1 2024-11-14T02:46:55,011 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-14T02:46:55,012 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T02:46:55,014 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-14T02:46:55,014 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731552475014 2024-11-14T02:46:55,014 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731552535014 2024-11-14T02:46:55,015 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-14T02:46:55,015 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,43215,1731552413607-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:55,015 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,43215,1731552413607-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:55,015 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,43215,1731552413607-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:55,015 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e5dc8b5cf5e9:43215, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:55,015 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:55,016 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:55,019 DEBUG [master/e5dc8b5cf5e9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T02:46:55,023 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.113sec 2024-11-14T02:46:55,024 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T02:46:55,024 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T02:46:55,024 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T02:46:55,024 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T02:46:55,024 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T02:46:55,024 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,43215,1731552413607-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T02:46:55,024 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,43215,1731552413607-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T02:46:55,025 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T02:46:55,025 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T02:46:55,025 INFO [master/e5dc8b5cf5e9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,43215,1731552413607-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:55,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17ad7caa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:55,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e5dc8b5cf5e9,43215,-1 for getting cluster id 2024-11-14T02:46:55,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T02:46:55,071 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83dfadfb-2547-4d23-aa00-fb6cbbf7b8fb' 2024-11-14T02:46:55,071 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T02:46:55,072 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83dfadfb-2547-4d23-aa00-fb6cbbf7b8fb" 2024-11-14T02:46:55,072 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2187318b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:55,072 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e5dc8b5cf5e9,43215,-1] 2024-11-14T02:46:55,073 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T02:46:55,073 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:55,075 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T02:46:55,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b03a5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T02:46:55,077 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T02:46:55,078 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e5dc8b5cf5e9,35911,1731552413798, seqNum=-1] 2024-11-14T02:46:55,079 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:46:55,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:46:55,083 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e5dc8b5cf5e9,43215,1731552413607 2024-11-14T02:46:55,084 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T02:46:55,086 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e5dc8b5cf5e9,43215,1731552413607 2024-11-14T02:46:55,086 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@39d7d8b3 2024-11-14T02:46:55,086 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T02:46:55,088 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T02:46:55,089 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731552415084', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T02:46:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731552415084 2024-11-14T02:46:55,091 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731552415084 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T02:46:55,091 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:55,091 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731552415084" procId is: 4 2024-11-14T02:46:55,092 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731552415084 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T02:46:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:46:55,101 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731552415084-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 847e687679deb9be1a1e9b4a6200ac2f, NAME => 'testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731552415084', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414 2024-11-14T02:46:55,107 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731552415084-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:55,107 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731552415084-pool-0 {}] regionserver.HRegion(1722): Closing 847e687679deb9be1a1e9b4a6200ac2f, disabling compactions & flushes 2024-11-14T02:46:55,107 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731552415084-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:55,107 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731552415084-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:55,107 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731552415084-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. after waiting 0 ms 2024-11-14T02:46:55,107 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731552415084-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:55,108 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731552415084-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:55,108 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731552415084-pool-0 {}] regionserver.HRegion(1676): Region close journal for 847e687679deb9be1a1e9b4a6200ac2f: Waiting for close lock at 1731552415107Disabling compacts and flushes for region at 1731552415107Disabling writes for close at 1731552415107Writing region close event to WAL at 1731552415107Closed at 1731552415108 (+1 ms) 2024-11-14T02:46:55,109 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731552415084 execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T02:46:55,109 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731552415109"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731552415109"}]},"ts":"1731552415109"} 2024-11-14T02:46:55,112 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T02:46:55,113 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731552415084 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T02:46:55,113 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731552415084","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552415113"}]},"ts":"1731552415113"} 2024-11-14T02:46:55,115 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731552415084, state=ENABLING in hbase:meta 2024-11-14T02:46:55,116 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {e5dc8b5cf5e9=0} racks are {/default-rack=0} 2024-11-14T02:46:55,116 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T02:46:55,116 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T02:46:55,116 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T02:46:55,116 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T02:46:55,116 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T02:46:55,116 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T02:46:55,116 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T02:46:55,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731552415084, region=847e687679deb9be1a1e9b4a6200ac2f, ASSIGN}] 2024-11-14T02:46:55,118 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731552415084, region=847e687679deb9be1a1e9b4a6200ac2f, ASSIGN 2024-11-14T02:46:55,119 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731552415084, region=847e687679deb9be1a1e9b4a6200ac2f, ASSIGN; state=OFFLINE, location=e5dc8b5cf5e9,33277,1731552413844; forceNewPlan=false, retain=false 2024-11-14T02:46:55,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:46:55,269 INFO [e5dc8b5cf5e9:43215 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T02:46:55,270 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=847e687679deb9be1a1e9b4a6200ac2f, regionState=OPENING, regionLocation=e5dc8b5cf5e9,33277,1731552413844 2024-11-14T02:46:55,272 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731552415084, region=847e687679deb9be1a1e9b4a6200ac2f, ASSIGN because future has completed 2024-11-14T02:46:55,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 847e687679deb9be1a1e9b4a6200ac2f, server=e5dc8b5cf5e9,33277,1731552413844}] 2024-11-14T02:46:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:46:55,425 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T02:46:55,427 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45927, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T02:46:55,430 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:55,431 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 847e687679deb9be1a1e9b4a6200ac2f, NAME => 'testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:46:55,431 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731552415084 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:55,431 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:55,431 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:55,431 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:55,443 INFO [StoreOpener-847e687679deb9be1a1e9b4a6200ac2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:55,450 INFO [StoreOpener-847e687679deb9be1a1e9b4a6200ac2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 847e687679deb9be1a1e9b4a6200ac2f columnFamilyName family 2024-11-14T02:46:55,450 DEBUG [StoreOpener-847e687679deb9be1a1e9b4a6200ac2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:55,450 INFO [StoreOpener-847e687679deb9be1a1e9b4a6200ac2f-1 {}] regionserver.HStore(327): Store=847e687679deb9be1a1e9b4a6200ac2f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:55,450 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:55,451 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:55,451 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:55,451 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:55,451 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:55,452 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:55,466 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T02:46:55,467 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 847e687679deb9be1a1e9b4a6200ac2f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60435158, jitterRate=-0.09944596886634827}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T02:46:55,467 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:55,468 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 847e687679deb9be1a1e9b4a6200ac2f: Running coprocessor pre-open hook at 1731552415441Writing region info on filesystem at 1731552415441Initializing all the Stores at 1731552415442 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552415442Cleaning up temporary data from old regions at 1731552415451 (+9 ms)Running coprocessor post-open hooks at 1731552415467 (+16 ms)Region opened successfully at 1731552415468 (+1 ms) 2024-11-14T02:46:55,470 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f., pid=6, masterSystemTime=1731552415425 2024-11-14T02:46:55,474 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:55,474 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=847e687679deb9be1a1e9b4a6200ac2f, regionState=OPEN, openSeqNum=2, regionLocation=e5dc8b5cf5e9,33277,1731552413844 2024-11-14T02:46:55,474 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:55,478 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 847e687679deb9be1a1e9b4a6200ac2f, server=e5dc8b5cf5e9,33277,1731552413844 because future has completed 2024-11-14T02:46:55,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T02:46:55,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 847e687679deb9be1a1e9b4a6200ac2f, server=e5dc8b5cf5e9,33277,1731552413844 in 207 msec 2024-11-14T02:46:55,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T02:46:55,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731552415084, region=847e687679deb9be1a1e9b4a6200ac2f, ASSIGN in 367 msec 2024-11-14T02:46:55,487 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731552415084 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T02:46:55,487 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731552415084","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731552415487"}]},"ts":"1731552415487"} 2024-11-14T02:46:55,490 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731552415084, state=ENABLED in hbase:meta 2024-11-14T02:46:55,492 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731552415084 execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T02:46:55,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731552415084 in 403 msec 2024-11-14T02:46:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T02:46:55,722 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731552415084 completed 2024-11-14T02:46:55,722 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731552415084 get assigned. Timeout = 60000ms 2024-11-14T02:46:55,723 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T02:46:55,727 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731552415084 assigned to meta. Checking AM states. 2024-11-14T02:46:55,728 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T02:46:55,728 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731552415084 assigned. 2024-11-14T02:46:55,733 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731552415084', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f., hostname=e5dc8b5cf5e9,33277,1731552413844, seqNum=2] 2024-11-14T02:46:55,733 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T02:46:55,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T02:46:55,746 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 847e687679deb9be1a1e9b4a6200ac2f 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-14T02:46:55,762 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 311cce8a2f5547028e8f7b47d14c93e9 2024-11-14T02:46:55,781 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/.tmp/family/311cce8a2f5547028e8f7b47d14c93e9 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-14T02:46:55,782 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/.tmp/family/311cce8a2f5547028e8f7b47d14c93e9 2024-11-14T02:46:55,785 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T02:46:55,786 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/.tmp/family/311cce8a2f5547028e8f7b47d14c93e9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9 2024-11-14T02:46:55,788 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9, delay=921 ms 2024-11-14T02:46:55,790 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9, entries=2, sequenceid=6, filesize=4.9 K 2024-11-14T02:46:55,794 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 847e687679deb9be1a1e9b4a6200ac2f in 48ms, sequenceid=6, compaction requested=false 2024-11-14T02:46:55,794 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731552415084' 2024-11-14T02:46:55,795 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 847e687679deb9be1a1e9b4a6200ac2f: 2024-11-14T02:46:56,711 DEBUG [hfile-prefetch-1731552412997 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 311cce8a2f5547028e8f7b47d14c93e9_0. 2024-11-14T02:46:56,711 DEBUG [hfile-prefetch-1731552412997 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 311cce8a2f5547028e8f7b47d14c93e9_117. 2024-11-14T02:46:56,712 DEBUG [hfile-prefetch-1731552412997 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9, with total blocks 2, and data blocks 1 2024-11-14T02:46:56,712 DEBUG [hfile-prefetch-1731552412997 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-14T02:46:56,712 DEBUG [hfile-prefetch-1731552412997 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-14T02:46:56,712 DEBUG [hfile-prefetch-1731552412997 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9. Will sleep 100ms and try the verification again. 2024-11-14T02:46:56,795 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;e5dc8b5cf5e9:33277,5,FailOnTimeoutGroup] 2024-11-14T02:46:56,796 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e5dc8b5cf5e9,33277,1731552413844' ***** 2024-11-14T02:46:56,796 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-14T02:46:56,796 INFO [RS:1;e5dc8b5cf5e9:33277 {}] bucket.BucketCache(1739): Disabling cache 2024-11-14T02:46:56,812 DEBUG [hfile-prefetch-1731552412997 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9, with total blocks 2, and data blocks 1 2024-11-14T02:46:56,812 DEBUG [hfile-prefetch-1731552412997 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-14T02:46:56,813 DEBUG [hfile-prefetch-1731552412997 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-14T02:46:56,813 DEBUG [hfile-prefetch-1731552412997 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9. Will sleep 100ms and try the verification again. 2024-11-14T02:46:56,852 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T02:46:56,853 INFO [regionserver/e5dc8b5cf5e9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T02:46:56,913 DEBUG [hfile-prefetch-1731552412997 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9, with total blocks 2, and data blocks 1 2024-11-14T02:46:56,913 DEBUG [hfile-prefetch-1731552412997 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-14T02:46:56,913 DEBUG [hfile-prefetch-1731552412997 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-14T02:46:56,913 DEBUG [hfile-prefetch-1731552412997 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9. Will sleep 100ms and try the verification again. 2024-11-14T02:46:56,968 INFO [RS:1;e5dc8b5cf5e9:33277 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/bucket.persistence 2024-11-14T02:46:56,968 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-14T02:46:56,968 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-14T02:46:56,968 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-14T02:46:56,968 INFO [RS:1;e5dc8b5cf5e9:33277 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-14T02:46:56,968 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-14T02:46:56,978 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] bucket.BucketCache(1488): Thread RS:1;e5dc8b5cf5e9:33277 started persisting bucket cache to file 2024-11-14T02:46:56,979 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T02:46:56,979 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T02:46:56,981 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-14T02:46:56,981 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731552416978 2024-11-14T02:46:56,981 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] bucket.BucketCache(1510): Thread RS:1;e5dc8b5cf5e9:33277 finished persisting bucket cache to file, renaming 2024-11-14T02:46:56,982 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T02:46:56,982 INFO [RS:1;e5dc8b5cf5e9:33277 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T02:46:56,982 INFO [RS:1;e5dc8b5cf5e9:33277 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T02:46:56,982 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HRegionServer(3091): Received CLOSE for 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:56,982 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HRegionServer(959): stopping server e5dc8b5cf5e9,33277,1731552413844 2024-11-14T02:46:56,982 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T02:46:56,982 INFO [RS:1;e5dc8b5cf5e9:33277 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;e5dc8b5cf5e9:33277. 2024-11-14T02:46:56,982 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-14T02:46:56,982 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:56,982 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 847e687679deb9be1a1e9b4a6200ac2f, disabling compactions & flushes 2024-11-14T02:46:56,982 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:56,982 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:56,982 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T02:46:56,982 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:56,983 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HRegionServer(1325): Online Regions={847e687679deb9be1a1e9b4a6200ac2f=testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f.} 2024-11-14T02:46:56,983 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. after waiting 0 ms 2024-11-14T02:46:56,983 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HRegionServer(1351): Waiting on 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:56,983 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:56,983 DEBUG [StoreFileCloser-847e687679deb9be1a1e9b4a6200ac2f-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9 2024-11-14T02:46:56,983 WARN [hfile-prefetch-1731552412997 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2322) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2322) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 11 more 2024-11-14T02:46:56,984 DEBUG [hfile-prefetch-1731552412997 {}] hfile.PrefetchExecutor(121): Prefetch completed for 311cce8a2f5547028e8f7b47d14c93e9 2024-11-14T02:46:56,987 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-14T02:46:56,988 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:56,988 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 847e687679deb9be1a1e9b4a6200ac2f: Waiting for close lock at 1731552416982Running coprocessor pre-close hooks at 1731552416982Disabling compacts and flushes for region at 1731552416982Disabling writes for close at 1731552416983 (+1 ms)Writing region close event to WAL at 1731552416983Running coprocessor post-close hooks at 1731552416988 (+5 ms)Closed at 1731552416988 2024-11-14T02:46:56,988 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:57,020 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-14T02:46:57,020 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-14T02:46:57,020 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T02:46:57,020 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T02:46:57,022 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-14T02:46:57,022 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731552417020 2024-11-14T02:46:57,022 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-14T02:46:57,183 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HRegionServer(976): stopping server e5dc8b5cf5e9,33277,1731552413844; all regions closed. 2024-11-14T02:46:57,185 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/oldWALs 2024-11-14T02:46:57,185 INFO [RS:1;e5dc8b5cf5e9:33277 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e5dc8b5cf5e9%2C33277%2C1731552413844:(num 1731552414694) 2024-11-14T02:46:57,185 DEBUG [RS:1;e5dc8b5cf5e9:33277 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:57,185 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T02:46:57,186 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T02:46:57,186 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.ChoreService(370): Chore service for: regionserver/e5dc8b5cf5e9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T02:46:57,186 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T02:46:57,186 INFO [regionserver/e5dc8b5cf5e9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T02:46:57,186 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T02:46:57,186 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T02:46:57,186 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T02:46:57,187 INFO [RS:1;e5dc8b5cf5e9:33277 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33277 2024-11-14T02:46:57,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e5dc8b5cf5e9,33277,1731552413844 2024-11-14T02:46:57,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T02:46:57,270 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T02:46:57,299 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e5dc8b5cf5e9,33277,1731552413844] 2024-11-14T02:46:57,309 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e5dc8b5cf5e9,33277,1731552413844 already deleted, retry=false 2024-11-14T02:46:57,309 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of e5dc8b5cf5e9,33277,1731552413844 on e5dc8b5cf5e9,43215,1731552413607 2024-11-14T02:46:57,314 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure e5dc8b5cf5e9,33277,1731552413844, splitWal=true, meta=false 2024-11-14T02:46:57,316 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure e5dc8b5cf5e9,33277,1731552413844, splitWal=true, meta=false 2024-11-14T02:46:57,317 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for e5dc8b5cf5e9,33277,1731552413844 (carryingMeta=false) e5dc8b5cf5e9,33277,1731552413844/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@79a548d6[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-14T02:46:57,317 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): e5dc8b5cf5e9,33277,1731552413844 had 1 regions 2024-11-14T02:46:57,319 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure e5dc8b5cf5e9,33277,1731552413844, splitWal=true, meta=false, isMeta: false 2024-11-14T02:46:57,319 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,33277,1731552413844-splitting 2024-11-14T02:46:57,319 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e5dc8b5cf5e9:0 server-side Connection retries=45 2024-11-14T02:46:57,320 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:57,320 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,33277,1731552413844-splitting dir is empty, no logs to split. 2024-11-14T02:46:57,320 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:57,320 INFO [PEWorker-1 {}] master.SplitWALManager(105): e5dc8b5cf5e9,33277,1731552413844 WAL count=0, meta=false 2024-11-14T02:46:57,320 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T02:46:57,320 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T02:46:57,320 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T02:46:57,320 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T02:46:57,320 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T02:46:57,321 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45775 2024-11-14T02:46:57,322 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,33277,1731552413844-splitting dir is empty, no logs to split. 2024-11-14T02:46:57,322 INFO [PEWorker-1 {}] master.SplitWALManager(105): e5dc8b5cf5e9,33277,1731552413844 WAL count=0, meta=false 2024-11-14T02:46:57,322 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if e5dc8b5cf5e9,33277,1731552413844 WAL splitting is done? wals=0, meta=false 2024-11-14T02:46:57,322 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45775 connecting to ZooKeeper ensemble=127.0.0.1:51661 2024-11-14T02:46:57,323 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:57,323 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:57,323 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for e5dc8b5cf5e9,33277,1731552413844 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,33277,1731552413844-splitting does not exist 2024-11-14T02:46:57,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731552415084, region=847e687679deb9be1a1e9b4a6200ac2f, ASSIGN}] 2024-11-14T02:46:57,327 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731552415084, region=847e687679deb9be1a1e9b4a6200ac2f, ASSIGN 2024-11-14T02:46:57,328 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731552415084, region=847e687679deb9be1a1e9b4a6200ac2f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T02:46:57,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:457750x0, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T02:46:57,330 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45775-0x101373d03430003, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-14T02:46:57,331 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45775-0x101373d03430003 connected 2024-11-14T02:46:57,331 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-14T02:46:57,331 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T02:46:57,332 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/bucket.cache 2024-11-14T02:46:57,332 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-14T02:46:57,332 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-14T02:46:57,332 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-14T02:46:57,332 INFO [Thread-200 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-14T02:46:57,332 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-14T02:46:57,333 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T02:46:57,333 INFO [Thread-200 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-14T02:46:57,333 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45775-0x101373d03430003, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T02:46:57,334 INFO [Thread-200 {}] bucket.BucketCache(1706): Retrieved 0 of chunks with blockCount = 0. 2024-11-14T02:46:57,335 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45775-0x101373d03430003, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T02:46:57,335 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45775 2024-11-14T02:46:57,336 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45775 2024-11-14T02:46:57,336 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45775 2024-11-14T02:46:57,341 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45775 2024-11-14T02:46:57,341 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45775 2024-11-14T02:46:57,343 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.HRegionServer(746): ClusterId : 83dfadfb-2547-4d23-aa00-fb6cbbf7b8fb 2024-11-14T02:46:57,343 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T02:46:57,344 DEBUG [Thread-200 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-14T02:46:57,344 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-14T02:46:57,344 INFO [Thread-200 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 0 2024-11-14T02:46:57,344 INFO [Thread-200 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/bucket.persistence is complete. 2024-11-14T02:46:57,352 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T02:46:57,352 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T02:46:57,362 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T02:46:57,363 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd5a6cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e5dc8b5cf5e9/172.17.0.2:0 2024-11-14T02:46:57,380 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;e5dc8b5cf5e9:45775 2024-11-14T02:46:57,380 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T02:46:57,380 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T02:46:57,380 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T02:46:57,381 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.HRegionServer(2659): reportForDuty to master=e5dc8b5cf5e9,43215,1731552413607 with port=45775, startcode=1731552417319 2024-11-14T02:46:57,381 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T02:46:57,383 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56799, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T02:46:57,384 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43215 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e5dc8b5cf5e9,45775,1731552417319 2024-11-14T02:46:57,384 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43215 {}] master.ServerManager(517): Registering regionserver=e5dc8b5cf5e9,45775,1731552417319 2024-11-14T02:46:57,385 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414 2024-11-14T02:46:57,385 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-14T02:46:57,385 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T02:46:57,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T02:46:57,394 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] zookeeper.ZKUtil(111): regionserver:45775-0x101373d03430003, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e5dc8b5cf5e9,45775,1731552417319 2024-11-14T02:46:57,394 WARN [RS:2;e5dc8b5cf5e9:45775 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T02:46:57,394 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e5dc8b5cf5e9,45775,1731552417319] 2024-11-14T02:46:57,394 INFO [RS:2;e5dc8b5cf5e9:45775 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T02:46:57,394 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,45775,1731552417319 2024-11-14T02:46:57,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:57,399 INFO [RS:1;e5dc8b5cf5e9:33277 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T02:46:57,399 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T02:46:57,399 INFO [RS:1;e5dc8b5cf5e9:33277 {}] regionserver.HRegionServer(1031): Exiting; stopping=e5dc8b5cf5e9,33277,1731552413844; zookeeper connection closed. 2024-11-14T02:46:57,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33277-0x101373d03430002, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:57,399 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4d94a4d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4d94a4d 2024-11-14T02:46:57,401 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T02:46:57,401 INFO [RS:2;e5dc8b5cf5e9:45775 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T02:46:57,401 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:57,402 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T02:46:57,402 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T02:46:57,402 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:57,403 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:57,403 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:57,403 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:57,403 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:57,403 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:57,403 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e5dc8b5cf5e9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T02:46:57,403 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:57,403 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:57,403 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:57,403 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:57,403 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:57,403 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e5dc8b5cf5e9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T02:46:57,403 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e5dc8b5cf5e9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T02:46:57,403 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e5dc8b5cf5e9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T02:46:57,404 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:57,404 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:57,404 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:57,404 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:57,404 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:57,404 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,45775,1731552417319-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T02:46:57,422 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T02:46:57,422 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.ChoreService(168): Chore ScheduledChore name=e5dc8b5cf5e9,45775,1731552417319-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:57,422 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:57,422 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.Replication(171): e5dc8b5cf5e9,45775,1731552417319 started 2024-11-14T02:46:57,436 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T02:46:57,436 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.HRegionServer(1482): Serving as e5dc8b5cf5e9,45775,1731552417319, RpcServer on e5dc8b5cf5e9/172.17.0.2:45775, sessionid=0x101373d03430003 2024-11-14T02:46:57,436 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T02:46:57,436 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e5dc8b5cf5e9,45775,1731552417319 2024-11-14T02:46:57,436 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5dc8b5cf5e9,45775,1731552417319' 2024-11-14T02:46:57,436 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T02:46:57,437 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T02:46:57,438 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T02:46:57,438 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T02:46:57,438 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e5dc8b5cf5e9,45775,1731552417319 2024-11-14T02:46:57,438 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e5dc8b5cf5e9,45775,1731552417319' 2024-11-14T02:46:57,438 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T02:46:57,438 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T02:46:57,439 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T02:46:57,439 INFO [RS:2;e5dc8b5cf5e9:45775 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T02:46:57,439 INFO [RS:2;e5dc8b5cf5e9:45775 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T02:46:57,478 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=847e687679deb9be1a1e9b4a6200ac2f, regionState=OPENING, regionLocation=e5dc8b5cf5e9,35911,1731552413798 2024-11-14T02:46:57,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731552415084, region=847e687679deb9be1a1e9b4a6200ac2f, ASSIGN because future has completed 2024-11-14T02:46:57,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 847e687679deb9be1a1e9b4a6200ac2f, server=e5dc8b5cf5e9,35911,1731552413798}] 2024-11-14T02:46:57,539 INFO [RS:2;e5dc8b5cf5e9:45775 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T02:46:57,540 INFO [RS:2;e5dc8b5cf5e9:45775 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e5dc8b5cf5e9%2C45775%2C1731552417319, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,45775,1731552417319, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/oldWALs, maxLogs=55 2024-11-14T02:46:57,559 INFO [RS:2;e5dc8b5cf5e9:45775 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/WALs/e5dc8b5cf5e9,45775,1731552417319/e5dc8b5cf5e9%2C45775%2C1731552417319.1731552417541 2024-11-14T02:46:57,559 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T02:46:57,608 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T02:46:57,608 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T02:46:57,609 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T02:46:57,609 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T02:46:57,610 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T02:46:57,610 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T02:46:57,637 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:57,637 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 847e687679deb9be1a1e9b4a6200ac2f, NAME => 'testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f.', STARTKEY => '', ENDKEY => ''} 2024-11-14T02:46:57,637 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731552415084 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:57,638 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T02:46:57,638 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:57,638 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:57,639 INFO [StoreOpener-847e687679deb9be1a1e9b4a6200ac2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:57,640 INFO [StoreOpener-847e687679deb9be1a1e9b4a6200ac2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 847e687679deb9be1a1e9b4a6200ac2f columnFamilyName family 2024-11-14T02:46:57,640 DEBUG [StoreOpener-847e687679deb9be1a1e9b4a6200ac2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T02:46:57,642 DEBUG [StoreFileOpener-847e687679deb9be1a1e9b4a6200ac2f-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9, delay=900 ms 2024-11-14T02:46:57,644 DEBUG [StoreOpener-847e687679deb9be1a1e9b4a6200ac2f-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9 2024-11-14T02:46:57,644 INFO [StoreOpener-847e687679deb9be1a1e9b4a6200ac2f-1 {}] regionserver.HStore(327): Store=847e687679deb9be1a1e9b4a6200ac2f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T02:46:57,645 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:57,646 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:57,647 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:57,647 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:57,648 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:57,650 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:57,651 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 847e687679deb9be1a1e9b4a6200ac2f; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60414036, jitterRate=-0.09976071119308472}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T02:46:57,651 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:57,652 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 847e687679deb9be1a1e9b4a6200ac2f: Running coprocessor pre-open hook at 1731552417638Writing region info on filesystem at 1731552417638Initializing all the Stores at 1731552417638Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731552417638Cleaning up temporary data from old regions at 1731552417648 (+10 ms)Running coprocessor post-open hooks at 1731552417651 (+3 ms)Region opened successfully at 1731552417652 (+1 ms) 2024-11-14T02:46:57,653 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f., pid=9, masterSystemTime=1731552417634 2024-11-14T02:46:57,657 DEBUG [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:57,657 INFO [RS_OPEN_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:57,657 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=847e687679deb9be1a1e9b4a6200ac2f, regionState=OPEN, openSeqNum=10, regionLocation=e5dc8b5cf5e9,35911,1731552413798 2024-11-14T02:46:57,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 847e687679deb9be1a1e9b4a6200ac2f, server=e5dc8b5cf5e9,35911,1731552413798 because future has completed 2024-11-14T02:46:57,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-14T02:46:57,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 847e687679deb9be1a1e9b4a6200ac2f, server=e5dc8b5cf5e9,35911,1731552413798 in 180 msec 2024-11-14T02:46:57,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T02:46:57,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731552415084, region=847e687679deb9be1a1e9b4a6200ac2f, ASSIGN in 338 msec 2024-11-14T02:46:57,665 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server e5dc8b5cf5e9,33277,1731552413844 after splitting done 2024-11-14T02:46:57,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure e5dc8b5cf5e9,33277,1731552413844, splitWal=true, meta=false in 355 msec 2024-11-14T02:46:57,936 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T02:46:57,936 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:46:57,937 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:57,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:57,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:57,937 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T02:46:57,937 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T02:46:57,937 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1942244894, stopped=false 2024-11-14T02:46:57,937 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e5dc8b5cf5e9,43215,1731552413607 2024-11-14T02:46:57,978 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T02:46:57,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T02:46:57,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45775-0x101373d03430003, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T02:46:57,979 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:57,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:57,979 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T02:46:57,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45775-0x101373d03430003, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:57,979 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T02:46:57,980 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:57,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:57,980 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:46:57,980 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45775-0x101373d03430003, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:46:57,980 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T02:46:57,981 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e5dc8b5cf5e9,35911,1731552413798' ***** 2024-11-14T02:46:57,981 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T02:46:57,981 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e5dc8b5cf5e9,45775,1731552417319' ***** 2024-11-14T02:46:57,981 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T02:46:57,981 INFO [RS:0;e5dc8b5cf5e9:35911 {}] bucket.BucketCache(1739): Disabling cache 2024-11-14T02:46:57,982 INFO [RS:2;e5dc8b5cf5e9:45775 {}] bucket.BucketCache(1739): Disabling cache 2024-11-14T02:46:57,982 INFO [RS:0;e5dc8b5cf5e9:35911 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/bucket.persistence 2024-11-14T02:46:57,982 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-14T02:46:57,982 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-14T02:46:57,982 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-14T02:46:57,982 INFO [RS:2;e5dc8b5cf5e9:45775 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9c80ce74-7ee5-b1d8-6a05-fc6375d6d5bf/bucket.persistence 2024-11-14T02:46:57,982 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-14T02:46:57,982 INFO [RS:0;e5dc8b5cf5e9:35911 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-14T02:46:57,982 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-14T02:46:57,983 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-14T02:46:57,982 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-14T02:46:57,983 INFO [RS:2;e5dc8b5cf5e9:45775 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-14T02:46:57,983 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-14T02:46:57,992 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] bucket.BucketCache(1488): Thread RS:0;e5dc8b5cf5e9:35911 started persisting bucket cache to file 2024-11-14T02:46:57,992 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T02:46:57,993 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T02:46:57,993 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] bucket.BucketCache(1488): Thread RS:2;e5dc8b5cf5e9:45775 started persisting bucket cache to file 2024-11-14T02:46:57,993 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T02:46:57,993 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-14T02:46:57,995 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-14T02:46:57,995 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731552417992 2024-11-14T02:46:57,995 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] bucket.BucketCache(1510): Thread RS:0;e5dc8b5cf5e9:35911 finished persisting bucket cache to file, renaming 2024-11-14T02:46:57,995 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-14T02:46:57,995 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731552417993 2024-11-14T02:46:57,995 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] bucket.BucketCache(1510): Thread RS:2;e5dc8b5cf5e9:45775 finished persisting bucket cache to file, renaming 2024-11-14T02:46:57,995 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T02:46:57,996 INFO [RS:0;e5dc8b5cf5e9:35911 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T02:46:57,996 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T02:46:57,996 INFO [RS:0;e5dc8b5cf5e9:35911 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T02:46:57,996 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer(3091): Received CLOSE for 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:57,996 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer(959): stopping server e5dc8b5cf5e9,35911,1731552413798 2024-11-14T02:46:57,996 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T02:46:57,996 INFO [RS:0;e5dc8b5cf5e9:35911 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e5dc8b5cf5e9:35911. 2024-11-14T02:46:57,996 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-14T02:46:57,996 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:57,996 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:57,996 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 847e687679deb9be1a1e9b4a6200ac2f, disabling compactions & flushes 2024-11-14T02:46:57,996 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:57,996 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:57,996 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T02:46:57,996 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. after waiting 0 ms 2024-11-14T02:46:57,996 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T02:46:57,996 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:57,996 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T02:46:57,997 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T02:46:57,997 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-14T02:46:57,997 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 847e687679deb9be1a1e9b4a6200ac2f=testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f.} 2024-11-14T02:46:57,997 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 847e687679deb9be1a1e9b4a6200ac2f 2024-11-14T02:46:57,997 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-14T02:46:57,997 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T02:46:57,997 DEBUG [StoreFileCloser-847e687679deb9be1a1e9b4a6200ac2f-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/family/311cce8a2f5547028e8f7b47d14c93e9 2024-11-14T02:46:57,997 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T02:46:57,997 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T02:46:57,997 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T02:46:57,997 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T02:46:57,997 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-14T02:46:57,998 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T02:46:57,998 INFO [RS:2;e5dc8b5cf5e9:45775 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T02:46:57,998 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T02:46:57,998 INFO [RS:2;e5dc8b5cf5e9:45775 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T02:46:57,998 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.HRegionServer(959): stopping server e5dc8b5cf5e9,45775,1731552417319 2024-11-14T02:46:57,998 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T02:46:57,999 INFO [RS:2;e5dc8b5cf5e9:45775 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;e5dc8b5cf5e9:45775. 2024-11-14T02:46:57,999 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T02:46:57,999 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:57,999 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.HRegionServer(976): stopping server e5dc8b5cf5e9,45775,1731552417319; all regions closed. 2024-11-14T02:46:58,001 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/oldWALs 2024-11-14T02:46:58,001 INFO [RS:2;e5dc8b5cf5e9:45775 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e5dc8b5cf5e9%2C45775%2C1731552417319:(num 1731552417541) 2024-11-14T02:46:58,001 DEBUG [RS:2;e5dc8b5cf5e9:45775 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:58,001 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T02:46:58,001 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T02:46:58,002 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.ChoreService(370): Chore service for: regionserver/e5dc8b5cf5e9:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T02:46:58,002 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T02:46:58,002 INFO [regionserver/e5dc8b5cf5e9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T02:46:58,002 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T02:46:58,002 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T02:46:58,002 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T02:46:58,002 INFO [RS:2;e5dc8b5cf5e9:45775 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45775 2024-11-14T02:46:58,004 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/default/testBlockEvictionOnGracefulStop1731552415084/847e687679deb9be1a1e9b4a6200ac2f/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-14T02:46:58,005 INFO [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:58,005 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 847e687679deb9be1a1e9b4a6200ac2f: Waiting for close lock at 1731552417996Running coprocessor pre-close hooks at 1731552417996Disabling compacts and flushes for region at 1731552417996Disabling writes for close at 1731552417996Writing region close event to WAL at 1731552417997 (+1 ms)Running coprocessor post-close hooks at 1731552418005 (+8 ms)Closed at 1731552418005 2024-11-14T02:46:58,005 DEBUG [RS_CLOSE_REGION-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f. 2024-11-14T02:46:58,007 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 067bc808b60648d1a96710c1f4a20c04 2024-11-14T02:46:58,009 INFO [regionserver/e5dc8b5cf5e9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T02:46:58,020 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/info/067bc808b60648d1a96710c1f4a20c04 is 209, key is testBlockEvictionOnGracefulStop1731552415084,,1731552415089.847e687679deb9be1a1e9b4a6200ac2f./info:regioninfo/1731552417657/Put/seqid=0 2024-11-14T02:46:58,021 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/info/067bc808b60648d1a96710c1f4a20c04 2024-11-14T02:46:58,028 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 031018f1b6f24090bbe91bb2d8be897c 2024-11-14T02:46:58,041 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/ns/031018f1b6f24090bbe91bb2d8be897c is 43, key is default/ns:d/1731552415006/Put/seqid=0 2024-11-14T02:46:58,042 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/ns/031018f1b6f24090bbe91bb2d8be897c 2024-11-14T02:46:58,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T02:46:58,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45775-0x101373d03430003, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e5dc8b5cf5e9,45775,1731552417319 2024-11-14T02:46:58,046 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T02:46:58,047 INFO [regionserver/e5dc8b5cf5e9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T02:46:58,050 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 672ea63526564547b0d02530a28cddfc 2024-11-14T02:46:58,061 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/table/672ea63526564547b0d02530a28cddfc is 80, key is testBlockEvictionOnGracefulStop1731552415084/table:state/1731552415487/Put/seqid=0 2024-11-14T02:46:58,062 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/table/672ea63526564547b0d02530a28cddfc 2024-11-14T02:46:58,064 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/info/067bc808b60648d1a96710c1f4a20c04 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/info/067bc808b60648d1a96710c1f4a20c04 2024-11-14T02:46:58,065 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/info/067bc808b60648d1a96710c1f4a20c04, delay=993 ms 2024-11-14T02:46:58,065 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/info/067bc808b60648d1a96710c1f4a20c04, entries=14, sequenceid=13, filesize=7.5 K 2024-11-14T02:46:58,066 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/ns/031018f1b6f24090bbe91bb2d8be897c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/ns/031018f1b6f24090bbe91bb2d8be897c 2024-11-14T02:46:58,067 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/ns/031018f1b6f24090bbe91bb2d8be897c, delay=983 ms 2024-11-14T02:46:58,067 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/ns/031018f1b6f24090bbe91bb2d8be897c, entries=2, sequenceid=13, filesize=5.0 K 2024-11-14T02:46:58,068 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/.tmp/table/672ea63526564547b0d02530a28cddfc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/table/672ea63526564547b0d02530a28cddfc 2024-11-14T02:46:58,069 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/table/672ea63526564547b0d02530a28cddfc, delay=913 ms 2024-11-14T02:46:58,069 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/table/672ea63526564547b0d02530a28cddfc, entries=2, sequenceid=13, filesize=5.3 K 2024-11-14T02:46:58,070 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 73ms, sequenceid=13, compaction requested=false 2024-11-14T02:46:58,071 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/info/067bc808b60648d1a96710c1f4a20c04 2024-11-14T02:46:58,071 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/ns/031018f1b6f24090bbe91bb2d8be897c 2024-11-14T02:46:58,072 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/table/672ea63526564547b0d02530a28cddfc 2024-11-14T02:46:58,075 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-14T02:46:58,076 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T02:46:58,076 INFO [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T02:46:58,076 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731552417997Running coprocessor pre-close hooks at 1731552417997Disabling compacts and flushes for region at 1731552417997Disabling writes for close at 1731552417997Obtaining lock to block concurrent updates at 1731552417997Preparing flush snapshotting stores in 1588230740 at 1731552417997Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1731552417998 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731552417998Flushing 1588230740/info: creating writer at 1731552417998Flushing 1588230740/info: appending metadata at 1731552418020 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731552418020Flushing 1588230740/ns: creating writer at 1731552418023 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731552418040 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731552418040Flushing 1588230740/table: creating writer at 1731552418044 (+4 ms)Flushing 1588230740/table: appending metadata at 1731552418061 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731552418061Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b6f90d6: reopening flushed file at 1731552418063 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5da425eb: reopening flushed file at 1731552418066 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b6d61df: reopening flushed file at 1731552418067 (+1 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 73ms, sequenceid=13, compaction requested=false at 1731552418070 (+3 ms)Writing region close event to WAL at 1731552418072 (+2 ms)Running coprocessor post-close hooks at 1731552418075 (+3 ms)Closed at 1731552418076 (+1 ms) 2024-11-14T02:46:58,076 DEBUG [RS_CLOSE_META-regionserver/e5dc8b5cf5e9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T02:46:58,105 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e5dc8b5cf5e9,45775,1731552417319] 2024-11-14T02:46:58,127 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e5dc8b5cf5e9,45775,1731552417319 already deleted, retry=false 2024-11-14T02:46:58,127 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e5dc8b5cf5e9,45775,1731552417319 expired; onlineServers=1 2024-11-14T02:46:58,197 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer(976): stopping server e5dc8b5cf5e9,35911,1731552413798; all regions closed. 2024-11-14T02:46:58,201 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/oldWALs 2024-11-14T02:46:58,201 INFO [RS:0;e5dc8b5cf5e9:35911 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e5dc8b5cf5e9%2C35911%2C1731552413798.meta:.meta(num 1731552414955) 2024-11-14T02:46:58,204 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/oldWALs 2024-11-14T02:46:58,204 INFO [RS:0;e5dc8b5cf5e9:35911 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e5dc8b5cf5e9%2C35911%2C1731552413798:(num 1731552414690) 2024-11-14T02:46:58,204 DEBUG [RS:0;e5dc8b5cf5e9:35911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T02:46:58,205 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T02:46:58,205 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T02:46:58,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45775-0x101373d03430003, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:58,205 INFO [RS:2;e5dc8b5cf5e9:45775 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T02:46:58,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45775-0x101373d03430003, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:58,206 INFO [RS:2;e5dc8b5cf5e9:45775 {}] regionserver.HRegionServer(1031): Exiting; stopping=e5dc8b5cf5e9,45775,1731552417319; zookeeper connection closed. 2024-11-14T02:46:58,205 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.ChoreService(370): Chore service for: regionserver/e5dc8b5cf5e9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T02:46:58,206 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T02:46:58,206 INFO [regionserver/e5dc8b5cf5e9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T02:46:58,206 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@317b89f2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@317b89f2 2024-11-14T02:46:58,207 INFO [RS:0;e5dc8b5cf5e9:35911 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35911 2024-11-14T02:46:58,221 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e5dc8b5cf5e9,35911,1731552413798 2024-11-14T02:46:58,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T02:46:58,221 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T02:46:58,232 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e5dc8b5cf5e9,35911,1731552413798] 2024-11-14T02:46:58,242 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e5dc8b5cf5e9,35911,1731552413798 already deleted, retry=false 2024-11-14T02:46:58,242 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e5dc8b5cf5e9,35911,1731552413798 expired; onlineServers=0 2024-11-14T02:46:58,242 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e5dc8b5cf5e9,43215,1731552413607' ***** 2024-11-14T02:46:58,242 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T02:46:58,242 INFO [M:0;e5dc8b5cf5e9:43215 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T02:46:58,243 INFO [M:0;e5dc8b5cf5e9:43215 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T02:46:58,243 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T02:46:58,243 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T02:46:58,243 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T02:46:58,243 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.large.0-1731552414311 {}] cleaner.HFileCleaner(306): Exit Thread[master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.large.0-1731552414311,5,FailOnTimeoutGroup] 2024-11-14T02:46:58,243 DEBUG [master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.small.0-1731552414311 {}] cleaner.HFileCleaner(306): Exit Thread[master/e5dc8b5cf5e9:0:becomeActiveMaster-HFileCleaner.small.0-1731552414311,5,FailOnTimeoutGroup] 2024-11-14T02:46:58,243 INFO [M:0;e5dc8b5cf5e9:43215 {}] hbase.ChoreService(370): Chore service for: master/e5dc8b5cf5e9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T02:46:58,243 INFO [M:0;e5dc8b5cf5e9:43215 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T02:46:58,244 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] master.HMaster(1795): Stopping service threads 2024-11-14T02:46:58,244 INFO [M:0;e5dc8b5cf5e9:43215 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T02:46:58,244 INFO [M:0;e5dc8b5cf5e9:43215 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T02:46:58,244 INFO [M:0;e5dc8b5cf5e9:43215 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T02:46:58,244 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T02:46:58,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T02:46:58,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T02:46:58,256 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] zookeeper.ZKUtil(347): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T02:46:58,256 WARN [M:0;e5dc8b5cf5e9:43215 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T02:46:58,257 INFO [M:0;e5dc8b5cf5e9:43215 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/.lastflushedseqids 2024-11-14T02:46:58,262 INFO [M:0;e5dc8b5cf5e9:43215 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T02:46:58,262 INFO [M:0;e5dc8b5cf5e9:43215 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T02:46:58,262 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T02:46:58,262 INFO [M:0;e5dc8b5cf5e9:43215 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:58,262 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:58,262 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T02:46:58,262 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:58,262 INFO [M:0;e5dc8b5cf5e9:43215 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.81 KB heapSize=41.05 KB 2024-11-14T02:46:58,271 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a926589b09dd47b6be44b094c4a3c4d8 2024-11-14T02:46:58,287 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a926589b09dd47b6be44b094c4a3c4d8 is 82, key is hbase:meta,,1/info:regioninfo/1731552414984/Put/seqid=0 2024-11-14T02:46:58,288 INFO [M:0;e5dc8b5cf5e9:43215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a926589b09dd47b6be44b094c4a3c4d8 2024-11-14T02:46:58,294 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on aa428435e3c64667b96ec9daf5153bec 2024-11-14T02:46:58,307 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aa428435e3c64667b96ec9daf5153bec is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731552415494/Put/seqid=0 2024-11-14T02:46:58,308 INFO [M:0;e5dc8b5cf5e9:43215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.08 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aa428435e3c64667b96ec9daf5153bec 2024-11-14T02:46:58,315 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f3ca3a90a6394bdf8071b9800dd527ea 2024-11-14T02:46:58,328 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f3ca3a90a6394bdf8071b9800dd527ea is 69, key is e5dc8b5cf5e9,35911,1731552413798/rs:state/1731552414422/Put/seqid=0 2024-11-14T02:46:58,329 INFO [M:0;e5dc8b5cf5e9:43215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f3ca3a90a6394bdf8071b9800dd527ea 2024-11-14T02:46:58,330 INFO [M:0;e5dc8b5cf5e9:43215 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f3ca3a90a6394bdf8071b9800dd527ea 2024-11-14T02:46:58,331 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a926589b09dd47b6be44b094c4a3c4d8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a926589b09dd47b6be44b094c4a3c4d8 2024-11-14T02:46:58,332 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:58,332 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35911-0x101373d03430001, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:58,332 INFO [RS:0;e5dc8b5cf5e9:35911 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T02:46:58,332 INFO [RS:0;e5dc8b5cf5e9:35911 {}] regionserver.HRegionServer(1031): Exiting; stopping=e5dc8b5cf5e9,35911,1731552413798; zookeeper connection closed. 2024-11-14T02:46:58,333 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@595332 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@595332 2024-11-14T02:46:58,333 INFO [M:0;e5dc8b5cf5e9:43215 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a926589b09dd47b6be44b094c4a3c4d8, entries=8, sequenceid=83, filesize=5.5 K 2024-11-14T02:46:58,333 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-14T02:46:58,333 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aa428435e3c64667b96ec9daf5153bec as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/aa428435e3c64667b96ec9daf5153bec 2024-11-14T02:46:58,334 INFO [M:0;e5dc8b5cf5e9:43215 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/aa428435e3c64667b96ec9daf5153bec, entries=9, sequenceid=83, filesize=6.4 K 2024-11-14T02:46:58,335 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f3ca3a90a6394bdf8071b9800dd527ea as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f3ca3a90a6394bdf8071b9800dd527ea 2024-11-14T02:46:58,336 INFO [M:0;e5dc8b5cf5e9:43215 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f3ca3a90a6394bdf8071b9800dd527ea 2024-11-14T02:46:58,336 INFO [M:0;e5dc8b5cf5e9:43215 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5d1bc9eb-5846-8d21-84ac-94997e8cc679/af03e907-814e-36f2-7267-419c684c6414/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f3ca3a90a6394bdf8071b9800dd527ea, entries=3, sequenceid=83, filesize=5.3 K 2024-11-14T02:46:58,337 INFO [M:0;e5dc8b5cf5e9:43215 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 75ms, sequenceid=83, compaction requested=false 2024-11-14T02:46:58,339 INFO [M:0;e5dc8b5cf5e9:43215 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T02:46:58,339 DEBUG [M:0;e5dc8b5cf5e9:43215 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731552418262Disabling compacts and flushes for region at 1731552418262Disabling writes for close at 1731552418262Obtaining lock to block concurrent updates at 1731552418262Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731552418262Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33602, getHeapSize=41976, getOffHeapSize=0, getCellsCount=98 at 1731552418263 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731552418263Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731552418263Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731552418287 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731552418287Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731552418289 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731552418307 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731552418307Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731552418310 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731552418327 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731552418327Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a0e5c58: reopening flushed file at 1731552418330 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@566b3e91: reopening flushed file at 1731552418333 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@784aa364: reopening flushed file at 1731552418335 (+2 ms)Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 75ms, sequenceid=83, compaction requested=false at 1731552418337 (+2 ms)Writing region close event to WAL at 1731552418339 (+2 ms)Closed at 1731552418339 2024-11-14T02:46:58,340 INFO [M:0;e5dc8b5cf5e9:43215 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T02:46:58,340 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T02:46:58,340 INFO [M:0;e5dc8b5cf5e9:43215 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43215 2024-11-14T02:46:58,340 INFO [M:0;e5dc8b5cf5e9:43215 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T02:46:58,451 INFO [M:0;e5dc8b5cf5e9:43215 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T02:46:58,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:58,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43215-0x101373d03430000, quorum=127.0.0.1:51661, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T02:46:58,455 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T02:46:58,456 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-14T02:46:58,457 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T02:46:58,464 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 49) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=476 (was 447) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11545 (was 11068) - AvailableMemoryMB LEAK? -