2024-11-16 12:54:54,891 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-16 12:54:54,908 main DEBUG Took 0.015158 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 12:54:54,908 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 12:54:54,908 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 12:54:54,909 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 12:54:54,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:54:54,917 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 12:54:54,935 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:54:54,937 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:54:54,938 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:54:54,938 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:54:54,939 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:54:54,940 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:54:54,941 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:54:54,941 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:54:54,942 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:54:54,942 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:54:54,944 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:54:54,944 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:54:54,945 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:54:54,946 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:54:54,946 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:54:54,947 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:54:54,947 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:54:54,948 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:54:54,949 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:54:54,949 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:54:54,950 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:54:54,950 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:54:54,951 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:54:54,951 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 12:54:54,952 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:54:54,952 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 12:54:54,954 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 12:54:54,956 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 12:54:54,959 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 12:54:54,960 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 12:54:54,961 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 12:54:54,962 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 12:54:54,973 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 12:54:54,976 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 12:54:54,979 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 12:54:54,979 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 12:54:54,980 main DEBUG createAppenders(={Console}) 2024-11-16 12:54:54,980 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-16 12:54:54,981 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-16 12:54:54,981 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-16 12:54:54,982 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 12:54:54,982 main DEBUG OutputStream closed 2024-11-16 12:54:54,983 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 12:54:54,983 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 12:54:54,983 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-16 12:54:55,055 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 12:54:55,057 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 12:54:55,058 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 12:54:55,059 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 12:54:55,060 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 12:54:55,060 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 12:54:55,060 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 12:54:55,061 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 12:54:55,061 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 12:54:55,061 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 12:54:55,062 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 12:54:55,062 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 12:54:55,062 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 12:54:55,063 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 12:54:55,063 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 12:54:55,063 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 12:54:55,063 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 12:54:55,064 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 12:54:55,066 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 12:54:55,066 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-16 12:54:55,066 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 12:54:55,067 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-16T12:54:55,078 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-16 12:54:55,080 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 12:54:55,081 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T12:54:55,295 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b 2024-11-16T12:54:55,318 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=426, ProcessCount=11, AvailableMemoryMB=4408 2024-11-16T12:54:55,402 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T12:54:55,482 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:54:55,497 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/cluster_1771e13e-1113-f7ec-ce71-4e8248c93f74, deleteOnExit=true 2024-11-16T12:54:55,632 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/cluster_1771e13e-1113-f7ec-ce71-4e8248c93f74/zookeeper_0, clientPort=60743, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/cluster_1771e13e-1113-f7ec-ce71-4e8248c93f74/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/cluster_1771e13e-1113-f7ec-ce71-4e8248c93f74/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T12:54:55,646 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60743 2024-11-16T12:54:55,647 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:54:55,739 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a with version=8 2024-11-16T12:54:55,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/hbase-staging 2024-11-16T12:54:55,840 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T12:54:56,108 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:54:56,117 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:54:56,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:54:56,126 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:54:56,127 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:54:56,127 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:54:56,249 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T12:54:56,322 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T12:54:56,354 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T12:54:56,366 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T12:54:56,370 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:54:56,403 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 93658 (auto-detected) 2024-11-16T12:54:56,404 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T12:54:56,423 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35289 2024-11-16T12:54:56,443 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35289 connecting to ZooKeeper ensemble=127.0.0.1:60743 2024-11-16T12:54:56,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:352890x0, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:54:56,546 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35289-0x10144fffe3e0000 connected 2024-11-16T12:54:56,635 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:54:56,637 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:54:56,646 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:54:56,649 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a, hbase.cluster.distributed=false 2024-11-16T12:54:56,673 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:54:56,679 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35289 2024-11-16T12:54:56,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35289 2024-11-16T12:54:56,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35289 2024-11-16T12:54:56,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35289 2024-11-16T12:54:56,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35289 2024-11-16T12:54:56,765 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:54:56,767 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:54:56,767 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:54:56,767 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:54:56,767 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:54:56,768 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:54:56,770 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:54:56,773 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:54:56,774 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46755 2024-11-16T12:54:56,776 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46755 connecting to ZooKeeper ensemble=127.0.0.1:60743 2024-11-16T12:54:56,776 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:54:56,777 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:54:56,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467550x0, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:54:56,796 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46755-0x10144fffe3e0001 connected 2024-11-16T12:54:56,796 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:54:56,804 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:54:56,815 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/bucket.cache 2024-11-16T12:54:56,821 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T12:54:56,822 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T12:54:56,824 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T12:54:56,825 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T12:54:56,826 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-16T12:54:56,833 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T12:54:56,835 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T12:54:56,835 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/bucket.persistence is complete. 2024-11-16T12:54:56,840 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:54:56,843 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:54:56,848 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:54:56,850 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46755 2024-11-16T12:54:56,851 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46755 2024-11-16T12:54:56,851 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46755 2024-11-16T12:54:56,852 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46755 2024-11-16T12:54:56,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46755 2024-11-16T12:54:56,871 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:54:56,871 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:54:56,871 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:54:56,872 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:54:56,872 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:54:56,872 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:54:56,872 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:54:56,873 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:54:56,874 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38269 2024-11-16T12:54:56,876 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38269 connecting to ZooKeeper ensemble=127.0.0.1:60743 2024-11-16T12:54:56,876 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:54:56,877 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:54:56,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382690x0, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:54:56,905 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:382690x0, quorum=127.0.0.1:60743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:54:56,905 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38269-0x10144fffe3e0002 connected 2024-11-16T12:54:56,905 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:54:56,906 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/bucket.cache 2024-11-16T12:54:56,907 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T12:54:56,907 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T12:54:56,908 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T12:54:56,911 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T12:54:56,911 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T12:54:56,911 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-16T12:54:56,912 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:54:56,912 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T12:54:56,912 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/bucket.persistence is complete. 2024-11-16T12:54:56,914 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:54:56,916 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:54:56,919 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38269 2024-11-16T12:54:56,923 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38269 2024-11-16T12:54:56,924 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38269 2024-11-16T12:54:56,927 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38269 2024-11-16T12:54:56,927 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38269 2024-11-16T12:54:56,944 DEBUG [M:0;0450ab8807f5:35289 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0450ab8807f5:35289 2024-11-16T12:54:56,947 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0450ab8807f5,35289,1731761695945 2024-11-16T12:54:56,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:54:56,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:54:56,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:54:56,978 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0450ab8807f5,35289,1731761695945 2024-11-16T12:54:57,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:54:57,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:54:57,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:57,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:57,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:57,022 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:54:57,023 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0450ab8807f5,35289,1731761695945 from backup master directory 2024-11-16T12:54:57,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:54:57,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0450ab8807f5,35289,1731761695945 2024-11-16T12:54:57,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:54:57,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:54:57,035 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:54:57,036 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0450ab8807f5,35289,1731761695945 2024-11-16T12:54:57,038 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T12:54:57,039 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T12:54:57,047 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/hbase.id] with ID: 4054cfe8-8fea-4818-aef2-664ac6458278 2024-11-16T12:54:57,047 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/.tmp/hbase.id 2024-11-16T12:54:57,052 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/hbase.id] 2024-11-16T12:54:57,090 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:54:57,096 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T12:54:57,116 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-16T12:54:57,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:57,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:57,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:57,168 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:54:57,171 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T12:54:57,207 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T12:54:57,212 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:54:57,262 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store 2024-11-16T12:54:57,281 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T12:54:57,284 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:54:57,286 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:54:57,286 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:54:57,286 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:54:57,288 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:54:57,288 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:54:57,288 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:54:57,288 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761697285Disabling compacts and flushes for region at 1731761697285Disabling writes for close at 1731761697288 (+3 ms)Writing region close event to WAL at 1731761697288Closed at 1731761697288 2024-11-16T12:54:57,289 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/.initializing 2024-11-16T12:54:57,289 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/WALs/0450ab8807f5,35289,1731761695945 2024-11-16T12:54:57,303 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:54:57,316 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C35289%2C1731761695945, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/WALs/0450ab8807f5,35289,1731761695945, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/oldWALs, maxLogs=10 2024-11-16T12:54:57,351 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/WALs/0450ab8807f5,35289,1731761695945/0450ab8807f5%2C35289%2C1731761695945.1731761697320 2024-11-16T12:54:57,351 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T12:54:57,352 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:54:57,352 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:54:57,356 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:54:57,357 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:54:57,399 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:54:57,427 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T12:54:57,431 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:54:57,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:54:57,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:54:57,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T12:54:57,439 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:54:57,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:54:57,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:54:57,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T12:54:57,446 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:54:57,447 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:54:57,447 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:54:57,451 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T12:54:57,452 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:54:57,452 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:54:57,453 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:54:57,457 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:54:57,458 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:54:57,462 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:54:57,463 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:54:57,467 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:54:57,471 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:54:57,478 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:54:57,479 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64414823, jitterRate=-0.040144339203834534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:54:57,486 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731761697369Initializing all the Stores at 1731761697369Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761697370 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761697370Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761697371 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761697371Cleaning up temporary data from old regions at 1731761697463 (+92 ms)Region opened successfully at 1731761697486 (+23 ms) 2024-11-16T12:54:57,488 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T12:54:57,520 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@907be14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:54:57,554 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T12:54:57,568 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T12:54:57,568 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T12:54:57,571 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T12:54:57,572 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T12:54:57,578 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-16T12:54:57,578 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T12:54:57,606 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T12:54:57,618 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T12:54:57,670 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T12:54:57,673 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T12:54:57,676 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T12:54:57,686 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T12:54:57,689 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T12:54:57,696 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T12:54:57,703 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T12:54:57,705 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T12:54:57,717 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T12:54:57,738 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T12:54:57,745 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T12:54:57,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:54:57,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:54:57,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:54:57,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:57,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:57,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:57,767 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0450ab8807f5,35289,1731761695945, sessionid=0x10144fffe3e0000, setting cluster-up flag (Was=false) 2024-11-16T12:54:57,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:57,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:57,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:57,820 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T12:54:57,823 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,35289,1731761695945 2024-11-16T12:54:57,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:57,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:57,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:57,870 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T12:54:57,874 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,35289,1731761695945 2024-11-16T12:54:57,883 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T12:54:57,931 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.HRegionServer(746): ClusterId : 4054cfe8-8fea-4818-aef2-664ac6458278 2024-11-16T12:54:57,931 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer(746): ClusterId : 4054cfe8-8fea-4818-aef2-664ac6458278 2024-11-16T12:54:57,934 DEBUG [RS:1;0450ab8807f5:38269 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:54:57,934 DEBUG [RS:0;0450ab8807f5:46755 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:54:57,949 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T12:54:57,955 DEBUG [RS:0;0450ab8807f5:46755 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:54:57,955 DEBUG [RS:1;0450ab8807f5:38269 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:54:57,955 DEBUG [RS:0;0450ab8807f5:46755 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:54:57,955 DEBUG [RS:1;0450ab8807f5:38269 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:54:57,961 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T12:54:57,968 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T12:54:57,971 DEBUG [RS:0;0450ab8807f5:46755 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:54:57,971 DEBUG [RS:1;0450ab8807f5:38269 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:54:57,971 DEBUG [RS:0;0450ab8807f5:46755 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@308d8bcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:54:57,972 DEBUG [RS:1;0450ab8807f5:38269 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bd8e86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:54:57,975 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0450ab8807f5,35289,1731761695945 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T12:54:57,983 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:54:57,984 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:54:57,984 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:54:57,984 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:54:57,984 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0450ab8807f5:0, corePoolSize=10, maxPoolSize=10 2024-11-16T12:54:57,984 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:57,984 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:54:57,985 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:57,986 DEBUG [RS:0;0450ab8807f5:46755 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0450ab8807f5:46755 2024-11-16T12:54:57,988 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731761727988 2024-11-16T12:54:57,988 DEBUG [RS:1;0450ab8807f5:38269 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0450ab8807f5:38269 2024-11-16T12:54:57,989 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:54:57,989 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:54:57,989 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:54:57,989 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:54:57,989 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:54:57,990 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T12:54:57,990 DEBUG [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:54:57,990 DEBUG [RS:1;0450ab8807f5:38269 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:54:57,990 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T12:54:57,991 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T12:54:57,993 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,35289,1731761695945 with port=38269, startcode=1731761696870 2024-11-16T12:54:57,993 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,35289,1731761695945 with port=46755, startcode=1731761696733 2024-11-16T12:54:57,994 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T12:54:57,995 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T12:54:57,995 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:54:57,995 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T12:54:57,995 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T12:54:57,995 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T12:54:57,996 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,000 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T12:54:58,001 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T12:54:58,002 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T12:54:58,007 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T12:54:58,007 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T12:54:58,007 DEBUG [RS:0;0450ab8807f5:46755 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:54:58,007 DEBUG [RS:1;0450ab8807f5:38269 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:54:58,008 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761698008,5,FailOnTimeoutGroup] 2024-11-16T12:54:58,009 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761698009,5,FailOnTimeoutGroup] 2024-11-16T12:54:58,009 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,009 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T12:54:58,010 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,010 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,011 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T12:54:58,012 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a 2024-11-16T12:54:58,020 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:54:58,021 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T12:54:58,025 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T12:54:58,025 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:54:58,026 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:54:58,026 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T12:54:58,033 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T12:54:58,033 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:54:58,033 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:54:58,034 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T12:54:58,039 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T12:54:58,039 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:54:58,039 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:54:58,040 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T12:54:58,044 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T12:54:58,045 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:54:58,045 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:54:58,045 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T12:54:58,053 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740 2024-11-16T12:54:58,053 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740 2024-11-16T12:54:58,054 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T12:54:58,054 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T12:54:58,054 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:54:58,055 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T12:54:58,068 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:54:58,069 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62666805, jitterRate=-0.06619183719158173}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:54:58,074 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731761698020Initializing all the Stores at 1731761698020Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761698020Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761698021 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761698021Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761698021Cleaning up temporary data from old regions at 1731761698054 (+33 ms)Region opened successfully at 1731761698074 (+20 ms) 2024-11-16T12:54:58,074 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T12:54:58,074 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T12:54:58,074 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T12:54:58,075 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T12:54:58,075 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T12:54:58,077 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T12:54:58,077 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731761698074Disabling compacts and flushes for region at 1731761698074Disabling writes for close at 1731761698075 (+1 ms)Writing region close event to WAL at 1731761698076 (+1 ms)Closed at 1731761698076 2024-11-16T12:54:58,077 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44939, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:54:58,077 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44775, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:54:58,079 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:54:58,080 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T12:54:58,085 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35289 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,46755,1731761696733 2024-11-16T12:54:58,088 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35289 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,46755,1731761696733 2024-11-16T12:54:58,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T12:54:58,098 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T12:54:58,100 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35289 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,38269,1731761696870 2024-11-16T12:54:58,100 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35289 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,38269,1731761696870 2024-11-16T12:54:58,100 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T12:54:58,104 DEBUG [RS:1;0450ab8807f5:38269 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a 2024-11-16T12:54:58,104 DEBUG [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a 2024-11-16T12:54:58,104 DEBUG [RS:1;0450ab8807f5:38269 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T12:54:58,104 DEBUG [RS:1;0450ab8807f5:38269 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:54:58,104 DEBUG [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T12:54:58,104 DEBUG [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:54:58,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:54:58,169 DEBUG [RS:0;0450ab8807f5:46755 {}] zookeeper.ZKUtil(111): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,46755,1731761696733 2024-11-16T12:54:58,169 DEBUG [RS:1;0450ab8807f5:38269 {}] zookeeper.ZKUtil(111): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,38269,1731761696870 2024-11-16T12:54:58,170 WARN [RS:1;0450ab8807f5:38269 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:54:58,170 WARN [RS:0;0450ab8807f5:46755 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:54:58,170 INFO [RS:0;0450ab8807f5:46755 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:54:58,170 INFO [RS:1;0450ab8807f5:38269 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:54:58,170 DEBUG [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,46755,1731761696733 2024-11-16T12:54:58,170 DEBUG [RS:1;0450ab8807f5:38269 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,38269,1731761696870 2024-11-16T12:54:58,173 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,46755,1731761696733] 2024-11-16T12:54:58,173 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,38269,1731761696870] 2024-11-16T12:54:58,198 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:54:58,198 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:54:58,216 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:54:58,216 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:54:58,221 INFO [RS:0;0450ab8807f5:46755 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:54:58,221 INFO [RS:1;0450ab8807f5:38269 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:54:58,221 INFO [RS:1;0450ab8807f5:38269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,221 INFO [RS:0;0450ab8807f5:46755 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,222 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:54:58,222 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:54:58,227 INFO [RS:1;0450ab8807f5:38269 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:54:58,227 INFO [RS:0;0450ab8807f5:46755 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:54:58,229 INFO [RS:0;0450ab8807f5:46755 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,229 INFO [RS:1;0450ab8807f5:38269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,229 DEBUG [RS:0;0450ab8807f5:46755 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,229 DEBUG [RS:1;0450ab8807f5:38269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,229 DEBUG [RS:0;0450ab8807f5:46755 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,229 DEBUG [RS:1;0450ab8807f5:38269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,229 DEBUG [RS:0;0450ab8807f5:46755 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,229 DEBUG [RS:1;0450ab8807f5:38269 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,229 DEBUG [RS:0;0450ab8807f5:46755 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,230 DEBUG [RS:1;0450ab8807f5:38269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,230 DEBUG [RS:0;0450ab8807f5:46755 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,230 DEBUG [RS:0;0450ab8807f5:46755 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:54:58,230 DEBUG [RS:1;0450ab8807f5:38269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,230 DEBUG [RS:0;0450ab8807f5:46755 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,230 DEBUG [RS:1;0450ab8807f5:38269 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:54:58,230 DEBUG [RS:0;0450ab8807f5:46755 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,230 DEBUG [RS:1;0450ab8807f5:38269 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,230 DEBUG [RS:0;0450ab8807f5:46755 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,230 DEBUG [RS:1;0450ab8807f5:38269 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,230 DEBUG [RS:0;0450ab8807f5:46755 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,230 DEBUG [RS:0;0450ab8807f5:46755 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,230 DEBUG [RS:1;0450ab8807f5:38269 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,231 DEBUG [RS:0;0450ab8807f5:46755 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,231 DEBUG [RS:1;0450ab8807f5:38269 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,231 DEBUG [RS:0;0450ab8807f5:46755 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:54:58,231 DEBUG [RS:0;0450ab8807f5:46755 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:54:58,231 DEBUG [RS:1;0450ab8807f5:38269 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,231 DEBUG [RS:1;0450ab8807f5:38269 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:54:58,231 DEBUG [RS:1;0450ab8807f5:38269 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:54:58,231 DEBUG [RS:1;0450ab8807f5:38269 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:54:58,233 INFO [RS:1;0450ab8807f5:38269 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,233 INFO [RS:1;0450ab8807f5:38269 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,233 INFO [RS:1;0450ab8807f5:38269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,234 INFO [RS:1;0450ab8807f5:38269 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,234 INFO [RS:1;0450ab8807f5:38269 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,234 INFO [RS:1;0450ab8807f5:38269 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,38269,1731761696870-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:54:58,235 INFO [RS:0;0450ab8807f5:46755 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,235 INFO [RS:0;0450ab8807f5:46755 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,235 INFO [RS:0;0450ab8807f5:46755 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,235 INFO [RS:0;0450ab8807f5:46755 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,236 INFO [RS:0;0450ab8807f5:46755 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,236 INFO [RS:0;0450ab8807f5:46755 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,46755,1731761696733-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:54:58,252 WARN [0450ab8807f5:35289 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-16T12:54:58,255 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:54:58,257 INFO [RS:1;0450ab8807f5:38269 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,38269,1731761696870-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,257 INFO [RS:1;0450ab8807f5:38269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,257 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.Replication(171): 0450ab8807f5,38269,1731761696870 started 2024-11-16T12:54:58,258 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:54:58,258 INFO [RS:0;0450ab8807f5:46755 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,46755,1731761696733-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,259 INFO [RS:0;0450ab8807f5:46755 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,259 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.Replication(171): 0450ab8807f5,46755,1731761696733 started 2024-11-16T12:54:58,276 INFO [RS:1;0450ab8807f5:38269 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,276 INFO [RS:0;0450ab8807f5:46755 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,276 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,38269,1731761696870, RpcServer on 0450ab8807f5/172.17.0.2:38269, sessionid=0x10144fffe3e0002 2024-11-16T12:54:58,276 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,46755,1731761696733, RpcServer on 0450ab8807f5/172.17.0.2:46755, sessionid=0x10144fffe3e0001 2024-11-16T12:54:58,277 DEBUG [RS:1;0450ab8807f5:38269 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:54:58,277 DEBUG [RS:0;0450ab8807f5:46755 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:54:58,277 DEBUG [RS:1;0450ab8807f5:38269 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,38269,1731761696870 2024-11-16T12:54:58,277 DEBUG [RS:0;0450ab8807f5:46755 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,46755,1731761696733 2024-11-16T12:54:58,277 DEBUG [RS:1;0450ab8807f5:38269 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,38269,1731761696870' 2024-11-16T12:54:58,277 DEBUG [RS:0;0450ab8807f5:46755 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,46755,1731761696733' 2024-11-16T12:54:58,277 DEBUG [RS:1;0450ab8807f5:38269 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:54:58,277 DEBUG [RS:0;0450ab8807f5:46755 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:54:58,278 DEBUG [RS:1;0450ab8807f5:38269 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:54:58,278 DEBUG [RS:0;0450ab8807f5:46755 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:54:58,279 DEBUG [RS:1;0450ab8807f5:38269 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:54:58,279 DEBUG [RS:0;0450ab8807f5:46755 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:54:58,279 DEBUG [RS:1;0450ab8807f5:38269 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:54:58,279 DEBUG [RS:0;0450ab8807f5:46755 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:54:58,279 DEBUG [RS:1;0450ab8807f5:38269 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,38269,1731761696870 2024-11-16T12:54:58,279 DEBUG [RS:0;0450ab8807f5:46755 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,46755,1731761696733 2024-11-16T12:54:58,279 DEBUG [RS:1;0450ab8807f5:38269 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,38269,1731761696870' 2024-11-16T12:54:58,279 DEBUG [RS:0;0450ab8807f5:46755 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,46755,1731761696733' 2024-11-16T12:54:58,279 DEBUG [RS:1;0450ab8807f5:38269 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:54:58,279 DEBUG [RS:0;0450ab8807f5:46755 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:54:58,280 DEBUG [RS:1;0450ab8807f5:38269 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:54:58,280 DEBUG [RS:0;0450ab8807f5:46755 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:54:58,280 DEBUG [RS:1;0450ab8807f5:38269 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:54:58,281 DEBUG [RS:0;0450ab8807f5:46755 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:54:58,281 INFO [RS:1;0450ab8807f5:38269 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:54:58,281 INFO [RS:0;0450ab8807f5:46755 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:54:58,281 INFO [RS:1;0450ab8807f5:38269 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:54:58,281 INFO [RS:0;0450ab8807f5:46755 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:54:58,388 INFO [RS:1;0450ab8807f5:38269 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:54:58,388 INFO [RS:0;0450ab8807f5:46755 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:54:58,389 INFO [RS:1;0450ab8807f5:38269 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0450ab8807f5%2C38269%2C1731761696870, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,38269,1731761696870, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/oldWALs, maxLogs=55 2024-11-16T12:54:58,390 INFO [RS:0;0450ab8807f5:46755 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0450ab8807f5%2C46755%2C1731761696733, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,46755,1731761696733, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/oldWALs, maxLogs=55 2024-11-16T12:54:58,410 INFO [RS:1;0450ab8807f5:38269 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,38269,1731761696870/0450ab8807f5%2C38269%2C1731761696870.1731761698394 2024-11-16T12:54:58,411 INFO [RS:0;0450ab8807f5:46755 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,46755,1731761696733/0450ab8807f5%2C46755%2C1731761696733.1731761698394 2024-11-16T12:54:58,411 DEBUG [RS:1;0450ab8807f5:38269 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T12:54:58,411 DEBUG [RS:0;0450ab8807f5:46755 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T12:54:58,506 DEBUG [0450ab8807f5:35289 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-16T12:54:58,523 DEBUG [0450ab8807f5:35289 {}] balancer.BalancerClusterState(204): Hosts are {0450ab8807f5=0} racks are {/default-rack=0} 2024-11-16T12:54:58,526 DEBUG [0450ab8807f5:35289 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T12:54:58,526 DEBUG [0450ab8807f5:35289 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T12:54:58,526 DEBUG [0450ab8807f5:35289 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T12:54:58,526 DEBUG [0450ab8807f5:35289 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T12:54:58,526 INFO [0450ab8807f5:35289 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T12:54:58,527 INFO [0450ab8807f5:35289 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T12:54:58,527 DEBUG [0450ab8807f5:35289 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T12:54:58,534 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0450ab8807f5,46755,1731761696733 2024-11-16T12:54:58,539 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,46755,1731761696733, state=OPENING 2024-11-16T12:54:58,570 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T12:54:58,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:58,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:58,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:54:58,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:54:58,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:54:58,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:54:58,583 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T12:54:58,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0450ab8807f5,46755,1731761696733}] 2024-11-16T12:54:58,760 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:54:58,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42129, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:54:58,772 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T12:54:58,772 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:54:58,774 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T12:54:58,775 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0450ab8807f5%2C46755%2C1731761696733.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,46755,1731761696733, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/oldWALs, maxLogs=55 2024-11-16T12:54:58,795 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,46755,1731761696733/0450ab8807f5%2C46755%2C1731761696733.meta.1731761698778.meta 2024-11-16T12:54:58,795 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T12:54:58,796 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:54:58,798 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T12:54:58,801 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T12:54:58,806 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T12:54:58,809 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T12:54:58,810 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:54:58,810 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T12:54:58,810 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T12:54:58,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T12:54:58,813 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T12:54:58,813 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:54:58,813 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:54:58,813 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T12:54:58,814 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T12:54:58,814 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:54:58,815 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:54:58,815 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T12:54:58,815 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T12:54:58,816 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:54:58,816 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:54:58,816 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T12:54:58,816 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T12:54:58,817 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:54:58,817 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:54:58,817 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T12:54:58,818 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740 2024-11-16T12:54:58,819 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740 2024-11-16T12:54:58,819 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T12:54:58,819 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T12:54:58,819 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:54:58,821 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T12:54:58,821 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68469623, jitterRate=0.020276889204978943}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:54:58,821 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T12:54:58,823 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731761698811Writing region info on filesystem at 1731761698811Initializing all the Stores at 1731761698811Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761698811Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761698812 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761698812Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761698812Cleaning up temporary data from old regions at 1731761698819 (+7 ms)Running coprocessor post-open hooks at 1731761698821 (+2 ms)Region opened successfully at 1731761698823 (+2 ms) 2024-11-16T12:54:58,828 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731761698752 2024-11-16T12:54:58,838 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,46755,1731761696733 2024-11-16T12:54:58,840 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,46755,1731761696733, state=OPEN 2024-11-16T12:54:58,841 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T12:54:58,842 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T12:54:58,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:54:58,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:54:58,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:54:58,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:54:58,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:54:58,877 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:54:58,877 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0450ab8807f5,46755,1731761696733 2024-11-16T12:54:58,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T12:54:58,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0450ab8807f5,46755,1731761696733 in 292 msec 2024-11-16T12:54:58,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T12:54:58,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 796 msec 2024-11-16T12:54:58,887 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:54:58,888 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T12:54:58,905 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:54:58,906 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,46755,1731761696733, seqNum=-1] 2024-11-16T12:54:58,923 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:54:58,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58845, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:54:58,945 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0330 sec 2024-11-16T12:54:58,945 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731761698945, completionTime=-1 2024-11-16T12:54:58,949 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-16T12:54:58,949 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T12:54:58,973 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T12:54:58,973 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731761758973 2024-11-16T12:54:58,973 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731761818973 2024-11-16T12:54:58,973 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-16T12:54:58,976 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-16T12:54:58,982 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,35289,1731761695945-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,983 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,35289,1731761695945-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,983 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,35289,1731761695945-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,984 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0450ab8807f5:35289, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,985 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,986 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:58,991 DEBUG [master/0450ab8807f5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T12:54:59,014 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.978sec 2024-11-16T12:54:59,015 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T12:54:59,017 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T12:54:59,017 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T12:54:59,018 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T12:54:59,018 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T12:54:59,019 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,35289,1731761695945-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:54:59,019 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,35289,1731761695945-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T12:54:59,020 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T12:54:59,021 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T12:54:59,022 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,35289,1731761695945-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:54:59,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T12:54:59,065 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T12:54:59,065 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T12:54:59,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0450ab8807f5,35289,-1 for getting cluster id 2024-11-16T12:54:59,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T12:54:59,079 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4054cfe8-8fea-4818-aef2-664ac6458278' 2024-11-16T12:54:59,081 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T12:54:59,081 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4054cfe8-8fea-4818-aef2-664ac6458278" 2024-11-16T12:54:59,082 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@606225e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T12:54:59,082 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0450ab8807f5,35289,-1] 2024-11-16T12:54:59,084 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T12:54:59,085 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:54:59,087 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T12:54:59,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T12:54:59,090 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:54:59,097 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,46755,1731761696733, seqNum=-1] 2024-11-16T12:54:59,097 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:54:59,100 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:54:59,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0450ab8807f5,35289,1731761695945 2024-11-16T12:54:59,123 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T12:54:59,128 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 0450ab8807f5,35289,1731761695945 2024-11-16T12:54:59,129 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@711da267 2024-11-16T12:54:59,130 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T12:54:59,133 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T12:54:59,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35289 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731761699119', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:54:59,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35289 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731761699119 2024-11-16T12:54:59,145 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731761699119 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T12:54:59,147 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35289 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731761699119" procId is: 4 2024-11-16T12:54:59,148 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:54:59,150 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731761699119 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T12:54:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:54:59,164 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731761699119-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4aa2323222f15ac3d7bffafc7e190a9b, NAME => 'testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731761699119', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a 2024-11-16T12:54:59,172 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731761699119-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:54:59,172 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731761699119-pool-0 {}] regionserver.HRegion(1722): Closing 4aa2323222f15ac3d7bffafc7e190a9b, disabling compactions & flushes 2024-11-16T12:54:59,172 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731761699119-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:54:59,172 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731761699119-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:54:59,172 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731761699119-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. after waiting 0 ms 2024-11-16T12:54:59,172 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731761699119-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:54:59,172 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731761699119-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:54:59,172 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731761699119-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4aa2323222f15ac3d7bffafc7e190a9b: Waiting for close lock at 1731761699172Disabling compacts and flushes for region at 1731761699172Disabling writes for close at 1731761699172Writing region close event to WAL at 1731761699172Closed at 1731761699172 2024-11-16T12:54:59,174 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731761699119 execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T12:54:59,178 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731761699174"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761699174"}]},"ts":"1731761699174"} 2024-11-16T12:54:59,182 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T12:54:59,183 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731761699119 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T12:54:59,186 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731761699119","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731761699183"}]},"ts":"1731761699183"} 2024-11-16T12:54:59,189 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731761699119, state=ENABLING in hbase:meta 2024-11-16T12:54:59,190 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {0450ab8807f5=0} racks are {/default-rack=0} 2024-11-16T12:54:59,191 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T12:54:59,191 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T12:54:59,191 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T12:54:59,191 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T12:54:59,191 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T12:54:59,191 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T12:54:59,191 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T12:54:59,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731761699119, region=4aa2323222f15ac3d7bffafc7e190a9b, ASSIGN}] 2024-11-16T12:54:59,194 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731761699119, region=4aa2323222f15ac3d7bffafc7e190a9b, ASSIGN 2024-11-16T12:54:59,195 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731761699119, region=4aa2323222f15ac3d7bffafc7e190a9b, ASSIGN; state=OFFLINE, location=0450ab8807f5,38269,1731761696870; forceNewPlan=false, retain=false 2024-11-16T12:54:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:54:59,351 INFO [0450ab8807f5:35289 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T12:54:59,353 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4aa2323222f15ac3d7bffafc7e190a9b, regionState=OPENING, regionLocation=0450ab8807f5,38269,1731761696870 2024-11-16T12:54:59,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731761699119, region=4aa2323222f15ac3d7bffafc7e190a9b, ASSIGN because future has completed 2024-11-16T12:54:59,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4aa2323222f15ac3d7bffafc7e190a9b, server=0450ab8807f5,38269,1731761696870}] 2024-11-16T12:54:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:54:59,514 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:54:59,518 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33569, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:54:59,523 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:54:59,523 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4aa2323222f15ac3d7bffafc7e190a9b, NAME => 'testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:54:59,525 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731761699119 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:54:59,525 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:54:59,525 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:54:59,526 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:54:59,527 INFO [StoreOpener-4aa2323222f15ac3d7bffafc7e190a9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:54:59,532 INFO [StoreOpener-4aa2323222f15ac3d7bffafc7e190a9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4aa2323222f15ac3d7bffafc7e190a9b columnFamilyName family 2024-11-16T12:54:59,532 DEBUG [StoreOpener-4aa2323222f15ac3d7bffafc7e190a9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:54:59,533 INFO [StoreOpener-4aa2323222f15ac3d7bffafc7e190a9b-1 {}] regionserver.HStore(327): Store=4aa2323222f15ac3d7bffafc7e190a9b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:54:59,533 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:54:59,535 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:54:59,535 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:54:59,535 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:54:59,535 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:54:59,537 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:54:59,542 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:54:59,542 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4aa2323222f15ac3d7bffafc7e190a9b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68685405, jitterRate=0.023492291569709778}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:54:59,542 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:54:59,543 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4aa2323222f15ac3d7bffafc7e190a9b: Running coprocessor pre-open hook at 1731761699526Writing region info on filesystem at 1731761699526Initializing all the Stores at 1731761699527 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761699527Cleaning up temporary data from old regions at 1731761699535 (+8 ms)Running coprocessor post-open hooks at 1731761699542 (+7 ms)Region opened successfully at 1731761699543 (+1 ms) 2024-11-16T12:54:59,544 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b., pid=6, masterSystemTime=1731761699513 2024-11-16T12:54:59,547 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:54:59,547 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:54:59,547 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4aa2323222f15ac3d7bffafc7e190a9b, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,38269,1731761696870 2024-11-16T12:54:59,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4aa2323222f15ac3d7bffafc7e190a9b, server=0450ab8807f5,38269,1731761696870 because future has completed 2024-11-16T12:54:59,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T12:54:59,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4aa2323222f15ac3d7bffafc7e190a9b, server=0450ab8807f5,38269,1731761696870 in 193 msec 2024-11-16T12:54:59,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T12:54:59,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731761699119, region=4aa2323222f15ac3d7bffafc7e190a9b, ASSIGN in 362 msec 2024-11-16T12:54:59,557 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731761699119 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T12:54:59,557 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731761699119","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731761699557"}]},"ts":"1731761699557"} 2024-11-16T12:54:59,561 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731761699119, state=ENABLED in hbase:meta 2024-11-16T12:54:59,562 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731761699119 execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T12:54:59,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731761699119 in 422 msec 2024-11-16T12:54:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35289 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:54:59,786 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731761699119 completed 2024-11-16T12:54:59,786 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731761699119 get assigned. Timeout = 60000ms 2024-11-16T12:54:59,791 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T12:54:59,797 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731761699119 assigned to meta. Checking AM states. 2024-11-16T12:54:59,798 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T12:54:59,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731761699119 assigned. 2024-11-16T12:54:59,806 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731761699119', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b., hostname=0450ab8807f5,38269,1731761696870, seqNum=2] 2024-11-16T12:54:59,807 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:54:59,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:54:59,818 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4aa2323222f15ac3d7bffafc7e190a9b 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-16T12:54:59,840 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 44ab7fea5dc44b45b422e8f2bab1717c 2024-11-16T12:54:59,874 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/.tmp/family/44ab7fea5dc44b45b422e8f2bab1717c is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-16T12:54:59,878 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/.tmp/family/44ab7fea5dc44b45b422e8f2bab1717c 2024-11-16T12:54:59,910 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/.tmp/family/44ab7fea5dc44b45b422e8f2bab1717c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/family/44ab7fea5dc44b45b422e8f2bab1717c 2024-11-16T12:54:59,911 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/family/44ab7fea5dc44b45b422e8f2bab1717c, delay=965 ms 2024-11-16T12:54:59,913 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/family/44ab7fea5dc44b45b422e8f2bab1717c, entries=2, sequenceid=6, filesize=4.9 K 2024-11-16T12:54:59,918 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 4aa2323222f15ac3d7bffafc7e190a9b in 99ms, sequenceid=6, compaction requested=false 2024-11-16T12:54:59,919 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731761699119' 2024-11-16T12:54:59,920 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4aa2323222f15ac3d7bffafc7e190a9b: 2024-11-16T12:55:00,881 DEBUG [hfile-prefetch-1731761699911 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 44ab7fea5dc44b45b422e8f2bab1717c_0. 2024-11-16T12:55:00,890 DEBUG [hfile-prefetch-1731761699911 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 44ab7fea5dc44b45b422e8f2bab1717c_117. 2024-11-16T12:55:00,891 DEBUG [hfile-prefetch-1731761699911 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/family/44ab7fea5dc44b45b422e8f2bab1717c, with total blocks 2, and data blocks 1 2024-11-16T12:55:00,891 DEBUG [hfile-prefetch-1731761699911 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-16T12:55:00,892 DEBUG [hfile-prefetch-1731761699911 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/family/44ab7fea5dc44b45b422e8f2bab1717c, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-16T12:55:00,892 DEBUG [hfile-prefetch-1731761699911 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/family/44ab7fea5dc44b45b422e8f2bab1717c. Will sleep 100ms and try the verification again. 2024-11-16T12:55:00,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35289 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=4aa2323222f15ac3d7bffafc7e190a9b, source=0450ab8807f5,38269,1731761696870, destination=0450ab8807f5,46755,1731761696733, warming up region on 0450ab8807f5,46755,1731761696733 2024-11-16T12:55:00,941 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35289 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=4aa2323222f15ac3d7bffafc7e190a9b, source=0450ab8807f5,38269,1731761696870, destination=0450ab8807f5,46755,1731761696733, running balancer 2024-11-16T12:55:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35289 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731761699119, region=4aa2323222f15ac3d7bffafc7e190a9b, REOPEN/MOVE 2024-11-16T12:55:00,942 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46755 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46755 {}] regionserver.HRegion(7855): Warmup {ENCODED => 4aa2323222f15ac3d7bffafc7e190a9b, NAME => 'testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:55:00,942 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731761699119, region=4aa2323222f15ac3d7bffafc7e190a9b, REOPEN/MOVE 2024-11-16T12:55:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46755 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:55:00,943 INFO [StoreOpener-4aa2323222f15ac3d7bffafc7e190a9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:55:00,944 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=4aa2323222f15ac3d7bffafc7e190a9b, regionState=CLOSING, regionLocation=0450ab8807f5,38269,1731761696870 2024-11-16T12:55:00,944 INFO [StoreOpener-4aa2323222f15ac3d7bffafc7e190a9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4aa2323222f15ac3d7bffafc7e190a9b columnFamilyName family 2024-11-16T12:55:00,944 DEBUG [StoreOpener-4aa2323222f15ac3d7bffafc7e190a9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:00,947 DEBUG [StoreOpener-4aa2323222f15ac3d7bffafc7e190a9b-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/family/44ab7fea5dc44b45b422e8f2bab1717c 2024-11-16T12:55:00,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731761699119, region=4aa2323222f15ac3d7bffafc7e190a9b, REOPEN/MOVE because future has completed 2024-11-16T12:55:00,947 INFO [StoreOpener-4aa2323222f15ac3d7bffafc7e190a9b-1 {}] regionserver.HStore(327): Store=4aa2323222f15ac3d7bffafc7e190a9b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:55:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46755 {}] regionserver.HRegion(1722): Closing 4aa2323222f15ac3d7bffafc7e190a9b, disabling compactions & flushes 2024-11-16T12:55:00,948 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46755 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46755 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46755 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. after waiting 0 ms 2024-11-16T12:55:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46755 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:00,948 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-16T12:55:00,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4aa2323222f15ac3d7bffafc7e190a9b, server=0450ab8807f5,38269,1731761696870}] 2024-11-16T12:55:00,949 DEBUG [StoreFileCloser-4aa2323222f15ac3d7bffafc7e190a9b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/family/44ab7fea5dc44b45b422e8f2bab1717c 2024-11-16T12:55:00,950 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46755 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46755 {}] regionserver.HRegion(1676): Region close journal for 4aa2323222f15ac3d7bffafc7e190a9b: Waiting for close lock at 1731761700948Disabling compacts and flushes for region at 1731761700948Disabling writes for close at 1731761700948Writing region close event to WAL at 1731761700949 (+1 ms)Closed at 1731761700950 (+1 ms) 2024-11-16T12:55:00,951 WARN [hfile-prefetch-1731761699911 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/family/44ab7fea5dc44b45b422e8f2bab1717c, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-16T12:55:00,952 DEBUG [hfile-prefetch-1731761699911 {}] hfile.PrefetchExecutor(121): Prefetch completed for 44ab7fea5dc44b45b422e8f2bab1717c 2024-11-16T12:55:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35289 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-16T12:55:01,014 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-16T12:55:01,014 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-16T12:55:01,015 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T12:55:01,015 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T12:55:01,028 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-16T12:55:01,028 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-16T12:55:01,028 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T12:55:01,028 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T12:55:01,030 ERROR [bucket-cache-persister {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream Closed at java.io.FileInputStream.available0(Native Method) ~[?:?] at java.io.FileInputStream.available(FileInputStream.java:415) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:351) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2024-11-16T12:55:01,032 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T12:55:01,032 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731761701028 2024-11-16T12:55:01,032 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-16T12:55:01,034 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-16T12:55:01,034 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731761701014 2024-11-16T12:55:01,034 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-16T12:55:01,107 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:55:01,107 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-16T12:55:01,108 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 4aa2323222f15ac3d7bffafc7e190a9b, disabling compactions & flushes 2024-11-16T12:55:01,108 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:01,108 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:01,108 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. after waiting 0 ms 2024-11-16T12:55:01,108 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:01,109 DEBUG [StoreFileCloser-4aa2323222f15ac3d7bffafc7e190a9b-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 44ab7fea5dc44b45b422e8f2bab1717c, starting offset: 0, end offset: 9223372036854775807 2024-11-16T12:55:01,110 DEBUG [StoreFileCloser-4aa2323222f15ac3d7bffafc7e190a9b-family-1 {}] bucket.BucketCache(856): removed key 44ab7fea5dc44b45b422e8f2bab1717c_0 from back map with offset lock 2094080 in the evict process 2024-11-16T12:55:01,115 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-16T12:55:01,116 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:01,116 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 4aa2323222f15ac3d7bffafc7e190a9b: Waiting for close lock at 1731761701107Running coprocessor pre-close hooks at 1731761701107Disabling compacts and flushes for region at 1731761701107Disabling writes for close at 1731761701108 (+1 ms)Writing region close event to WAL at 1731761701111 (+3 ms)Running coprocessor post-close hooks at 1731761701115 (+4 ms)Closed at 1731761701116 (+1 ms) 2024-11-16T12:55:01,116 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 4aa2323222f15ac3d7bffafc7e190a9b move to 0450ab8807f5,46755,1731761696733 record at close sequenceid=6 2024-11-16T12:55:01,119 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:55:01,120 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=4aa2323222f15ac3d7bffafc7e190a9b, regionState=CLOSED 2024-11-16T12:55:01,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4aa2323222f15ac3d7bffafc7e190a9b, server=0450ab8807f5,38269,1731761696870 because future has completed 2024-11-16T12:55:01,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T12:55:01,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 4aa2323222f15ac3d7bffafc7e190a9b, server=0450ab8807f5,38269,1731761696870 in 176 msec 2024-11-16T12:55:01,126 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731761699119, region=4aa2323222f15ac3d7bffafc7e190a9b, REOPEN/MOVE; state=CLOSED, location=0450ab8807f5,46755,1731761696733; forceNewPlan=false, retain=false 2024-11-16T12:55:01,132 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-16T12:55:01,132 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-16T12:55:01,132 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T12:55:01,132 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T12:55:01,135 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T12:55:01,135 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731761701132 2024-11-16T12:55:01,135 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-16T12:55:01,277 INFO [0450ab8807f5:35289 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T12:55:01,278 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=4aa2323222f15ac3d7bffafc7e190a9b, regionState=OPENING, regionLocation=0450ab8807f5,46755,1731761696733 2024-11-16T12:55:01,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731761699119, region=4aa2323222f15ac3d7bffafc7e190a9b, REOPEN/MOVE because future has completed 2024-11-16T12:55:01,286 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4aa2323222f15ac3d7bffafc7e190a9b, server=0450ab8807f5,46755,1731761696733}] 2024-11-16T12:55:01,445 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:01,446 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 4aa2323222f15ac3d7bffafc7e190a9b, NAME => 'testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:55:01,446 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731761699119 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:55:01,446 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:55:01,447 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:55:01,447 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:55:01,448 INFO [StoreOpener-4aa2323222f15ac3d7bffafc7e190a9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:55:01,449 INFO [StoreOpener-4aa2323222f15ac3d7bffafc7e190a9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4aa2323222f15ac3d7bffafc7e190a9b columnFamilyName family 2024-11-16T12:55:01,449 DEBUG [StoreOpener-4aa2323222f15ac3d7bffafc7e190a9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:01,451 DEBUG [StoreFileOpener-4aa2323222f15ac3d7bffafc7e190a9b-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/family/44ab7fea5dc44b45b422e8f2bab1717c, delay=962 ms 2024-11-16T12:55:01,453 DEBUG [StoreOpener-4aa2323222f15ac3d7bffafc7e190a9b-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/family/44ab7fea5dc44b45b422e8f2bab1717c 2024-11-16T12:55:01,453 INFO [StoreOpener-4aa2323222f15ac3d7bffafc7e190a9b-1 {}] regionserver.HStore(327): Store=4aa2323222f15ac3d7bffafc7e190a9b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:55:01,453 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:55:01,454 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:55:01,455 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:55:01,455 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:55:01,455 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:55:01,460 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:55:01,461 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 4aa2323222f15ac3d7bffafc7e190a9b; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66454384, jitterRate=-0.009752511978149414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:55:01,461 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:55:01,462 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 4aa2323222f15ac3d7bffafc7e190a9b: Running coprocessor pre-open hook at 1731761701447Writing region info on filesystem at 1731761701447Initializing all the Stores at 1731761701447Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761701448 (+1 ms)Cleaning up temporary data from old regions at 1731761701455 (+7 ms)Running coprocessor post-open hooks at 1731761701461 (+6 ms)Region opened successfully at 1731761701462 (+1 ms) 2024-11-16T12:55:01,463 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b., pid=9, masterSystemTime=1731761701440 2024-11-16T12:55:01,465 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:01,466 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:01,466 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=4aa2323222f15ac3d7bffafc7e190a9b, regionState=OPEN, openSeqNum=10, regionLocation=0450ab8807f5,46755,1731761696733 2024-11-16T12:55:01,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4aa2323222f15ac3d7bffafc7e190a9b, server=0450ab8807f5,46755,1731761696733 because future has completed 2024-11-16T12:55:01,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-16T12:55:01,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 4aa2323222f15ac3d7bffafc7e190a9b, server=0450ab8807f5,46755,1731761696733 in 184 msec 2024-11-16T12:55:01,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731761699119, region=4aa2323222f15ac3d7bffafc7e190a9b, REOPEN/MOVE in 531 msec 2024-11-16T12:55:01,495 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T12:55:01,496 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T12:55:01,496 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:55:01,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:01,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:01,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T12:55:01,502 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T12:55:01,503 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-16T12:55:01,503 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0450ab8807f5,35289,1731761695945 2024-11-16T12:55:01,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T12:55:01,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T12:55:01,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T12:55:01,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:01,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:01,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:01,518 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T12:55:01,518 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T12:55:01,519 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:55:01,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:01,519 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:55:01,519 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:55:01,519 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:55:01,520 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,46755,1731761696733' ***** 2024-11-16T12:55:01,520 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T12:55:01,520 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,38269,1731761696870' ***** 2024-11-16T12:55:01,520 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T12:55:01,521 INFO [RS:0;0450ab8807f5:46755 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T12:55:01,521 INFO [RS:1;0450ab8807f5:38269 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T12:55:01,521 INFO [RS:0;0450ab8807f5:46755 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/bucket.persistence 2024-11-16T12:55:01,521 INFO [RS:1;0450ab8807f5:38269 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/bucket.persistence 2024-11-16T12:55:01,521 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T12:55:01,521 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T12:55:01,522 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T12:55:01,522 INFO [RS:1;0450ab8807f5:38269 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T12:55:01,522 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T12:55:01,522 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T12:55:01,522 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T12:55:01,522 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T12:55:01,523 INFO [RS:0;0450ab8807f5:46755 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T12:55:01,523 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T12:55:01,532 DEBUG [RS:1;0450ab8807f5:38269 {}] bucket.BucketCache(1487): Thread RS:1;0450ab8807f5:38269 started persisting bucket cache to file 2024-11-16T12:55:01,532 DEBUG [RS:1;0450ab8807f5:38269 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T12:55:01,532 DEBUG [RS:1;0450ab8807f5:38269 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T12:55:01,533 DEBUG [RS:0;0450ab8807f5:46755 {}] bucket.BucketCache(1487): Thread RS:0;0450ab8807f5:46755 started persisting bucket cache to file 2024-11-16T12:55:01,533 DEBUG [RS:0;0450ab8807f5:46755 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T12:55:01,533 DEBUG [RS:0;0450ab8807f5:46755 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T12:55:01,535 DEBUG [RS:0;0450ab8807f5:46755 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T12:55:01,535 DEBUG [RS:0;0450ab8807f5:46755 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731761701533 2024-11-16T12:55:01,535 DEBUG [RS:0;0450ab8807f5:46755 {}] bucket.BucketCache(1509): Thread RS:0;0450ab8807f5:46755 finished persisting bucket cache to file, renaming 2024-11-16T12:55:01,535 DEBUG [RS:1;0450ab8807f5:38269 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T12:55:01,536 DEBUG [RS:1;0450ab8807f5:38269 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731761701532 2024-11-16T12:55:01,536 DEBUG [RS:1;0450ab8807f5:38269 {}] bucket.BucketCache(1509): Thread RS:1;0450ab8807f5:38269 finished persisting bucket cache to file, renaming 2024-11-16T12:55:01,536 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:55:01,536 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:55:01,536 INFO [RS:0;0450ab8807f5:46755 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T12:55:01,536 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:55:01,536 INFO [RS:1;0450ab8807f5:38269 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T12:55:01,536 INFO [RS:1;0450ab8807f5:38269 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T12:55:01,536 INFO [RS:0;0450ab8807f5:46755 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T12:55:01,536 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.HRegionServer(959): stopping server 0450ab8807f5,38269,1731761696870 2024-11-16T12:55:01,536 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer(3091): Received CLOSE for 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:55:01,536 INFO [RS:1;0450ab8807f5:38269 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:55:01,537 INFO [RS:1;0450ab8807f5:38269 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0450ab8807f5:38269. 2024-11-16T12:55:01,537 DEBUG [RS:1;0450ab8807f5:38269 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:55:01,537 DEBUG [RS:1;0450ab8807f5:38269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:01,537 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer(959): stopping server 0450ab8807f5,46755,1731761696733 2024-11-16T12:55:01,537 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.HRegionServer(976): stopping server 0450ab8807f5,38269,1731761696870; all regions closed. 2024-11-16T12:55:01,537 INFO [RS:0;0450ab8807f5:46755 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:55:01,537 INFO [RS:0;0450ab8807f5:46755 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0450ab8807f5:46755. 2024-11-16T12:55:01,537 DEBUG [RS:0;0450ab8807f5:46755 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:55:01,537 DEBUG [RS:0;0450ab8807f5:46755 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:01,538 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:55:01,538 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T12:55:01,538 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T12:55:01,538 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T12:55:01,538 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T12:55:01,538 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T12:55:01,538 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4aa2323222f15ac3d7bffafc7e190a9b, disabling compactions & flushes 2024-11-16T12:55:01,538 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:01,538 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:01,538 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-16T12:55:01,538 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. after waiting 0 ms 2024-11-16T12:55:01,538 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T12:55:01,538 DEBUG [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 4aa2323222f15ac3d7bffafc7e190a9b=testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b.} 2024-11-16T12:55:01,538 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:01,538 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T12:55:01,538 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T12:55:01,539 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T12:55:01,539 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T12:55:01,539 DEBUG [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4aa2323222f15ac3d7bffafc7e190a9b 2024-11-16T12:55:01,539 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T12:55:01,539 DEBUG [StoreFileCloser-4aa2323222f15ac3d7bffafc7e190a9b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/family/44ab7fea5dc44b45b422e8f2bab1717c 2024-11-16T12:55:01,539 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-16T12:55:01,541 DEBUG [RS:1;0450ab8807f5:38269 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/oldWALs 2024-11-16T12:55:01,541 INFO [RS:1;0450ab8807f5:38269 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0450ab8807f5%2C38269%2C1731761696870:(num 1731761698394) 2024-11-16T12:55:01,541 DEBUG [RS:1;0450ab8807f5:38269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:01,541 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:55:01,541 INFO [RS:1;0450ab8807f5:38269 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:55:01,542 INFO [RS:1;0450ab8807f5:38269 {}] hbase.ChoreService(370): Chore service for: regionserver/0450ab8807f5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T12:55:01,542 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T12:55:01,542 INFO [regionserver/0450ab8807f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:55:01,542 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T12:55:01,542 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T12:55:01,542 INFO [RS:1;0450ab8807f5:38269 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:55:01,543 INFO [RS:1;0450ab8807f5:38269 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38269 2024-11-16T12:55:01,543 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnRegionMove1731761699119/4aa2323222f15ac3d7bffafc7e190a9b/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-16T12:55:01,544 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:01,544 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4aa2323222f15ac3d7bffafc7e190a9b: Waiting for close lock at 1731761701537Running coprocessor pre-close hooks at 1731761701538 (+1 ms)Disabling compacts and flushes for region at 1731761701538Disabling writes for close at 1731761701538Writing region close event to WAL at 1731761701539 (+1 ms)Running coprocessor post-close hooks at 1731761701544 (+5 ms)Closed at 1731761701544 2024-11-16T12:55:01,544 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b. 2024-11-16T12:55:01,548 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 872bbb8ff5704ea3862a17ad5542ed59 2024-11-16T12:55:01,552 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:55:01,556 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:55:01,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0450ab8807f5,38269,1731761696870 2024-11-16T12:55:01,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:55:01,561 INFO [RS:1;0450ab8807f5:38269 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:55:01,564 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/info/872bbb8ff5704ea3862a17ad5542ed59 is 205, key is testBlockEvictionOnRegionMove1731761699119,,1731761699134.4aa2323222f15ac3d7bffafc7e190a9b./info:regioninfo/1731761701466/Put/seqid=0 2024-11-16T12:55:01,564 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/info/872bbb8ff5704ea3862a17ad5542ed59 2024-11-16T12:55:01,569 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0450ab8807f5,38269,1731761696870] 2024-11-16T12:55:01,572 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 02bc95fcb61c43a6833daf7060751288 2024-11-16T12:55:01,577 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0450ab8807f5,38269,1731761696870 already deleted, retry=false 2024-11-16T12:55:01,577 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0450ab8807f5,38269,1731761696870 expired; onlineServers=1 2024-11-16T12:55:01,586 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/ns/02bc95fcb61c43a6833daf7060751288 is 43, key is default/ns:d/1731761698929/Put/seqid=0 2024-11-16T12:55:01,587 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/ns/02bc95fcb61c43a6833daf7060751288 2024-11-16T12:55:01,596 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 655ed245705e4252b83eb61559528780 2024-11-16T12:55:01,609 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/table/655ed245705e4252b83eb61559528780 is 78, key is testBlockEvictionOnRegionMove1731761699119/table:state/1731761699557/Put/seqid=0 2024-11-16T12:55:01,609 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/table/655ed245705e4252b83eb61559528780 2024-11-16T12:55:01,612 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/info/872bbb8ff5704ea3862a17ad5542ed59 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/info/872bbb8ff5704ea3862a17ad5542ed59 2024-11-16T12:55:01,613 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/info/872bbb8ff5704ea3862a17ad5542ed59, delay=999 ms 2024-11-16T12:55:01,615 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/info/872bbb8ff5704ea3862a17ad5542ed59, entries=15, sequenceid=15, filesize=7.6 K 2024-11-16T12:55:01,615 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/ns/02bc95fcb61c43a6833daf7060751288 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/ns/02bc95fcb61c43a6833daf7060751288 2024-11-16T12:55:01,616 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/ns/02bc95fcb61c43a6833daf7060751288, delay=990 ms 2024-11-16T12:55:01,618 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/ns/02bc95fcb61c43a6833daf7060751288, entries=2, sequenceid=15, filesize=5.0 K 2024-11-16T12:55:01,618 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/table/655ed245705e4252b83eb61559528780 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/table/655ed245705e4252b83eb61559528780 2024-11-16T12:55:01,619 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/table/655ed245705e4252b83eb61559528780, delay=927 ms 2024-11-16T12:55:01,620 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/table/655ed245705e4252b83eb61559528780, entries=2, sequenceid=15, filesize=5.3 K 2024-11-16T12:55:01,621 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 82ms, sequenceid=15, compaction requested=false 2024-11-16T12:55:01,621 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T12:55:01,622 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/info/872bbb8ff5704ea3862a17ad5542ed59 2024-11-16T12:55:01,623 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/ns/02bc95fcb61c43a6833daf7060751288 2024-11-16T12:55:01,624 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/table/655ed245705e4252b83eb61559528780 2024-11-16T12:55:01,627 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-16T12:55:01,628 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T12:55:01,628 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T12:55:01,628 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731761701538Running coprocessor pre-close hooks at 1731761701538Disabling compacts and flushes for region at 1731761701538Disabling writes for close at 1731761701539 (+1 ms)Obtaining lock to block concurrent updates at 1731761701539Preparing flush snapshotting stores in 1588230740 at 1731761701539Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731761701540 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731761701540Flushing 1588230740/info: creating writer at 1731761701540Flushing 1588230740/info: appending metadata at 1731761701563 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731761701563Flushing 1588230740/ns: creating writer at 1731761701567 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731761701585 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731761701585Flushing 1588230740/table: creating writer at 1731761701590 (+5 ms)Flushing 1588230740/table: appending metadata at 1731761701608 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731761701608Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b0e335e: reopening flushed file at 1731761701612 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b727ed6: reopening flushed file at 1731761701615 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1967b8ab: reopening flushed file at 1731761701618 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 82ms, sequenceid=15, compaction requested=false at 1731761701621 (+3 ms)Writing region close event to WAL at 1731761701624 (+3 ms)Running coprocessor post-close hooks at 1731761701627 (+3 ms)Closed at 1731761701628 (+1 ms) 2024-11-16T12:55:01,628 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T12:55:01,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:55:01,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38269-0x10144fffe3e0002, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:55:01,670 INFO [RS:1;0450ab8807f5:38269 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:55:01,671 INFO [RS:1;0450ab8807f5:38269 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,38269,1731761696870; zookeeper connection closed. 2024-11-16T12:55:01,671 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@24d5853 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@24d5853 2024-11-16T12:55:01,739 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer(976): stopping server 0450ab8807f5,46755,1731761696733; all regions closed. 2024-11-16T12:55:01,741 DEBUG [RS:0;0450ab8807f5:46755 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/oldWALs 2024-11-16T12:55:01,741 INFO [RS:0;0450ab8807f5:46755 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0450ab8807f5%2C46755%2C1731761696733.meta:.meta(num 1731761698778) 2024-11-16T12:55:01,743 DEBUG [RS:0;0450ab8807f5:46755 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/oldWALs 2024-11-16T12:55:01,743 INFO [RS:0;0450ab8807f5:46755 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0450ab8807f5%2C46755%2C1731761696733:(num 1731761698394) 2024-11-16T12:55:01,743 DEBUG [RS:0;0450ab8807f5:46755 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:01,743 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:55:01,743 INFO [RS:0;0450ab8807f5:46755 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:55:01,744 INFO [RS:0;0450ab8807f5:46755 {}] hbase.ChoreService(370): Chore service for: regionserver/0450ab8807f5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T12:55:01,744 INFO [RS:0;0450ab8807f5:46755 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:55:01,744 INFO [regionserver/0450ab8807f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:55:01,744 INFO [RS:0;0450ab8807f5:46755 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46755 2024-11-16T12:55:01,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0450ab8807f5,46755,1731761696733 2024-11-16T12:55:01,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:55:01,761 INFO [RS:0;0450ab8807f5:46755 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:55:01,769 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0450ab8807f5,46755,1731761696733] 2024-11-16T12:55:01,777 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0450ab8807f5,46755,1731761696733 already deleted, retry=false 2024-11-16T12:55:01,777 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0450ab8807f5,46755,1731761696733 expired; onlineServers=0 2024-11-16T12:55:01,777 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0450ab8807f5,35289,1731761695945' ***** 2024-11-16T12:55:01,777 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T12:55:01,777 INFO [M:0;0450ab8807f5:35289 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:55:01,778 INFO [M:0;0450ab8807f5:35289 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:55:01,778 DEBUG [M:0;0450ab8807f5:35289 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T12:55:01,778 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T12:55:01,778 DEBUG [M:0;0450ab8807f5:35289 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T12:55:01,778 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761698008 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761698008,5,FailOnTimeoutGroup] 2024-11-16T12:55:01,778 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761698009 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761698009,5,FailOnTimeoutGroup] 2024-11-16T12:55:01,778 INFO [M:0;0450ab8807f5:35289 {}] hbase.ChoreService(370): Chore service for: master/0450ab8807f5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T12:55:01,778 INFO [M:0;0450ab8807f5:35289 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:55:01,779 DEBUG [M:0;0450ab8807f5:35289 {}] master.HMaster(1795): Stopping service threads 2024-11-16T12:55:01,779 INFO [M:0;0450ab8807f5:35289 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T12:55:01,779 INFO [M:0;0450ab8807f5:35289 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T12:55:01,779 INFO [M:0;0450ab8807f5:35289 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T12:55:01,780 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T12:55:01,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:55:01,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:01,786 DEBUG [M:0;0450ab8807f5:35289 {}] zookeeper.ZKUtil(347): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T12:55:01,786 WARN [M:0;0450ab8807f5:35289 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T12:55:01,786 INFO [M:0;0450ab8807f5:35289 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/.lastflushedseqids 2024-11-16T12:55:01,797 INFO [M:0;0450ab8807f5:35289 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T12:55:01,798 INFO [M:0;0450ab8807f5:35289 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T12:55:01,798 DEBUG [M:0;0450ab8807f5:35289 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:55:01,798 INFO [M:0;0450ab8807f5:35289 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:55:01,798 DEBUG [M:0;0450ab8807f5:35289 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:55:01,798 DEBUG [M:0;0450ab8807f5:35289 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:55:01,798 DEBUG [M:0;0450ab8807f5:35289 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:55:01,799 INFO [M:0;0450ab8807f5:35289 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-16T12:55:01,808 DEBUG [M:0;0450ab8807f5:35289 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 70e9b13deeba461c88be79e4a1ac73c4 2024-11-16T12:55:01,821 DEBUG [M:0;0450ab8807f5:35289 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70e9b13deeba461c88be79e4a1ac73c4 is 82, key is hbase:meta,,1/info:regioninfo/1731761698838/Put/seqid=0 2024-11-16T12:55:01,821 INFO [M:0;0450ab8807f5:35289 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70e9b13deeba461c88be79e4a1ac73c4 2024-11-16T12:55:01,829 DEBUG [M:0;0450ab8807f5:35289 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a847b3281e844474bcb0d677ee6eceeb 2024-11-16T12:55:01,842 DEBUG [M:0;0450ab8807f5:35289 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a847b3281e844474bcb0d677ee6eceeb is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731761699563/Put/seqid=0 2024-11-16T12:55:01,842 INFO [M:0;0450ab8807f5:35289 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a847b3281e844474bcb0d677ee6eceeb 2024-11-16T12:55:01,850 DEBUG [M:0;0450ab8807f5:35289 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 07bce1a9a1ad456cb018969665f36957 2024-11-16T12:55:01,861 DEBUG [M:0;0450ab8807f5:35289 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/07bce1a9a1ad456cb018969665f36957 is 69, key is 0450ab8807f5,38269,1731761696870/rs:state/1731761698101/Put/seqid=0 2024-11-16T12:55:01,862 INFO [M:0;0450ab8807f5:35289 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/07bce1a9a1ad456cb018969665f36957 2024-11-16T12:55:01,864 DEBUG [M:0;0450ab8807f5:35289 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70e9b13deeba461c88be79e4a1ac73c4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/70e9b13deeba461c88be79e4a1ac73c4 2024-11-16T12:55:01,865 INFO [M:0;0450ab8807f5:35289 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/70e9b13deeba461c88be79e4a1ac73c4, entries=8, sequenceid=84, filesize=5.5 K 2024-11-16T12:55:01,865 DEBUG [M:0;0450ab8807f5:35289 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a847b3281e844474bcb0d677ee6eceeb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a847b3281e844474bcb0d677ee6eceeb 2024-11-16T12:55:01,867 INFO [M:0;0450ab8807f5:35289 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a847b3281e844474bcb0d677ee6eceeb, entries=9, sequenceid=84, filesize=6.5 K 2024-11-16T12:55:01,868 DEBUG [M:0;0450ab8807f5:35289 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/07bce1a9a1ad456cb018969665f36957 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/07bce1a9a1ad456cb018969665f36957 2024-11-16T12:55:01,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:55:01,869 INFO [RS:0;0450ab8807f5:46755 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:55:01,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46755-0x10144fffe3e0001, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:55:01,869 INFO [RS:0;0450ab8807f5:46755 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,46755,1731761696733; zookeeper connection closed. 2024-11-16T12:55:01,869 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@764aaec7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@764aaec7 2024-11-16T12:55:01,870 INFO [M:0;0450ab8807f5:35289 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/07bce1a9a1ad456cb018969665f36957, entries=2, sequenceid=84, filesize=5.1 K 2024-11-16T12:55:01,870 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-16T12:55:01,871 INFO [M:0;0450ab8807f5:35289 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 73ms, sequenceid=84, compaction requested=false 2024-11-16T12:55:01,872 INFO [M:0;0450ab8807f5:35289 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:55:01,873 DEBUG [M:0;0450ab8807f5:35289 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761701798Disabling compacts and flushes for region at 1731761701798Disabling writes for close at 1731761701798Obtaining lock to block concurrent updates at 1731761701799 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731761701799Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731761701799Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731761701800 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731761701800Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731761701820 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731761701820Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731761701823 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731761701841 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731761701841Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731761701844 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731761701861 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731761701861Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ea17dca: reopening flushed file at 1731761701863 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@581030e1: reopening flushed file at 1731761701865 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e309061: reopening flushed file at 1731761701867 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 73ms, sequenceid=84, compaction requested=false at 1731761701871 (+4 ms)Writing region close event to WAL at 1731761701872 (+1 ms)Closed at 1731761701872 2024-11-16T12:55:01,874 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:55:01,874 INFO [M:0;0450ab8807f5:35289 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T12:55:01,874 INFO [M:0;0450ab8807f5:35289 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35289 2024-11-16T12:55:01,875 INFO [M:0;0450ab8807f5:35289 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:55:01,992 INFO [M:0;0450ab8807f5:35289 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:55:01,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:55:01,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35289-0x10144fffe3e0000, quorum=127.0.0.1:60743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:55:01,999 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T12:55:02,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-16T12:55:02,020 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:55:02,028 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/0450ab8807f5:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/0450ab8807f5:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731761699911 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731761701616 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/0450ab8807f5:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/0450ab8807f5:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1731761701613 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1731761701451 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=408 (was 426), ProcessCount=11 (was 11), AvailableMemoryMB=4067 (was 4408) 2024-11-16T12:55:02,033 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=408, ProcessCount=11, AvailableMemoryMB=4066 2024-11-16T12:55:02,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/hadoop-log-dir so I do NOT create it in target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91 2024-11-16T12:55:02,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/hadoop-tmp-dir so I do NOT create it in target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91 2024-11-16T12:55:02,034 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:55:02,036 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/cluster_4e83e436-d8f5-cb9f-c624-7e067435e5b6, deleteOnExit=true 2024-11-16T12:55:02,039 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/cluster_4e83e436-d8f5-cb9f-c624-7e067435e5b6/zookeeper_0, clientPort=53805, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/cluster_4e83e436-d8f5-cb9f-c624-7e067435e5b6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/cluster_4e83e436-d8f5-cb9f-c624-7e067435e5b6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T12:55:02,040 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53805 2024-11-16T12:55:02,050 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a with version=8 2024-11-16T12:55:02,050 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/hbase-staging 2024-11-16T12:55:02,053 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:55:02,053 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:55:02,054 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:55:02,054 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:55:02,054 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:55:02,054 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:55:02,054 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T12:55:02,054 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:55:02,055 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38799 2024-11-16T12:55:02,057 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38799 connecting to ZooKeeper ensemble=127.0.0.1:53805 2024-11-16T12:55:02,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:387990x0, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:55:02,096 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38799-0x1014500175b0000 connected 2024-11-16T12:55:02,170 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:55:02,171 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:55:02,173 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:55:02,173 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a, hbase.cluster.distributed=false 2024-11-16T12:55:02,178 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:55:02,179 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38799 2024-11-16T12:55:02,180 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38799 2024-11-16T12:55:02,180 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38799 2024-11-16T12:55:02,181 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38799 2024-11-16T12:55:02,182 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38799 2024-11-16T12:55:02,199 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:55:02,199 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:55:02,199 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:55:02,199 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:55:02,199 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:55:02,199 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:55:02,199 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:55:02,199 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:55:02,200 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41647 2024-11-16T12:55:02,201 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41647 connecting to ZooKeeper ensemble=127.0.0.1:53805 2024-11-16T12:55:02,202 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:55:02,202 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:55:02,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:416470x0, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:55:02,212 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41647-0x1014500175b0001 connected 2024-11-16T12:55:02,212 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:55:02,212 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:55:02,213 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/bucket.cache 2024-11-16T12:55:02,213 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T12:55:02,213 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T12:55:02,213 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T12:55:02,213 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T12:55:02,214 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-16T12:55:02,214 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T12:55:02,214 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:55:02,214 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T12:55:02,214 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/bucket.persistence is complete. 2024-11-16T12:55:02,215 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:55:02,216 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:55:02,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41647 2024-11-16T12:55:02,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41647 2024-11-16T12:55:02,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41647 2024-11-16T12:55:02,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41647 2024-11-16T12:55:02,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41647 2024-11-16T12:55:02,235 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:55:02,236 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:55:02,236 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:55:02,236 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:55:02,236 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:55:02,236 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:55:02,236 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:55:02,236 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:55:02,237 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36895 2024-11-16T12:55:02,238 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36895 connecting to ZooKeeper ensemble=127.0.0.1:53805 2024-11-16T12:55:02,238 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:55:02,239 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:55:02,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:368950x0, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:55:02,251 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:55:02,251 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36895-0x1014500175b0002 connected 2024-11-16T12:55:02,251 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:55:02,252 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/bucket.cache 2024-11-16T12:55:02,252 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T12:55:02,252 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T12:55:02,252 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T12:55:02,253 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T12:55:02,253 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-16T12:55:02,253 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T12:55:02,253 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:55:02,253 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T12:55:02,253 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/bucket.persistence is complete. 2024-11-16T12:55:02,254 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T12:55:02,255 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:55:02,257 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36895 2024-11-16T12:55:02,258 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36895 2024-11-16T12:55:02,258 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36895 2024-11-16T12:55:02,259 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36895 2024-11-16T12:55:02,260 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36895 2024-11-16T12:55:02,271 DEBUG [M:0;0450ab8807f5:38799 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0450ab8807f5:38799 2024-11-16T12:55:02,272 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0450ab8807f5,38799,1731761702052 2024-11-16T12:55:02,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:55:02,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:55:02,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:55:02,278 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0450ab8807f5,38799,1731761702052 2024-11-16T12:55:02,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:55:02,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T12:55:02,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:02,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:02,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:02,287 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:55:02,287 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0450ab8807f5,38799,1731761702052 from backup master directory 2024-11-16T12:55:02,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0450ab8807f5,38799,1731761702052 2024-11-16T12:55:02,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:55:02,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:55:02,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T12:55:02,294 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:55:02,295 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0450ab8807f5,38799,1731761702052 2024-11-16T12:55:02,296 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/hbase.id] with ID: 0ef62761-1acb-404b-8483-845645e00a64 2024-11-16T12:55:02,296 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/.tmp/hbase.id 2024-11-16T12:55:02,301 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/hbase.id] 2024-11-16T12:55:02,334 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:55:02,334 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T12:55:02,338 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 4ms. 2024-11-16T12:55:02,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:02,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:02,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:02,355 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:55:02,356 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T12:55:02,357 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:55:02,376 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store 2024-11-16T12:55:02,383 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:55:02,384 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:55:02,384 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:55:02,384 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:55:02,384 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:55:02,384 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:55:02,384 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:55:02,384 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761702384Disabling compacts and flushes for region at 1731761702384Disabling writes for close at 1731761702384Writing region close event to WAL at 1731761702384Closed at 1731761702384 2024-11-16T12:55:02,385 WARN [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/.initializing 2024-11-16T12:55:02,385 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/WALs/0450ab8807f5,38799,1731761702052 2024-11-16T12:55:02,392 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:55:02,396 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0450ab8807f5%2C38799%2C1731761702052, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/WALs/0450ab8807f5,38799,1731761702052, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/oldWALs, maxLogs=10 2024-11-16T12:55:02,412 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/WALs/0450ab8807f5,38799,1731761702052/0450ab8807f5%2C38799%2C1731761702052.1731761702396 2024-11-16T12:55:02,413 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T12:55:02,413 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:55:02,413 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:55:02,413 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:55:02,413 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:55:02,414 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:55:02,419 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T12:55:02,419 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:02,419 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:55:02,420 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:55:02,423 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T12:55:02,423 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:02,423 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:55:02,423 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:55:02,426 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T12:55:02,426 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:02,426 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:55:02,427 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:55:02,430 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T12:55:02,430 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:02,430 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:55:02,430 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:55:02,431 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:55:02,431 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:55:02,431 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:55:02,431 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:55:02,432 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:55:02,432 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T12:55:02,437 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:55:02,438 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71278078, jitterRate=0.06212612986564636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:55:02,439 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731761702414Initializing all the Stores at 1731761702414Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761702414Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761702414Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761702414Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761702414Cleaning up temporary data from old regions at 1731761702431 (+17 ms)Region opened successfully at 1731761702439 (+8 ms) 2024-11-16T12:55:02,439 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T12:55:02,443 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6245513f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:55:02,444 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T12:55:02,444 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T12:55:02,444 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T12:55:02,445 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T12:55:02,445 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T12:55:02,445 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-16T12:55:02,445 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T12:55:02,448 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T12:55:02,449 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T12:55:02,475 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T12:55:02,476 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T12:55:02,477 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T12:55:02,486 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T12:55:02,487 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T12:55:02,488 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T12:55:02,494 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T12:55:02,495 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T12:55:02,502 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T12:55:02,505 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T12:55:02,511 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T12:55:02,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:55:02,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:55:02,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:02,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:02,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T12:55:02,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:02,520 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0450ab8807f5,38799,1731761702052, sessionid=0x1014500175b0000, setting cluster-up flag (Was=false) 2024-11-16T12:55:02,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:02,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:02,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:02,561 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T12:55:02,563 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,38799,1731761702052 2024-11-16T12:55:02,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:02,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:02,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:02,603 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T12:55:02,604 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0450ab8807f5,38799,1731761702052 2024-11-16T12:55:02,604 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T12:55:02,606 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T12:55:02,606 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T12:55:02,606 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T12:55:02,606 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0450ab8807f5,38799,1731761702052 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T12:55:02,608 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:55:02,608 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:55:02,608 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:55:02,608 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0450ab8807f5:0, corePoolSize=5, maxPoolSize=5 2024-11-16T12:55:02,608 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0450ab8807f5:0, corePoolSize=10, maxPoolSize=10 2024-11-16T12:55:02,608 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,608 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:55:02,608 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,609 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731761732609 2024-11-16T12:55:02,610 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T12:55:02,610 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T12:55:02,610 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T12:55:02,610 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:55:02,610 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T12:55:02,610 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T12:55:02,610 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T12:55:02,610 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T12:55:02,610 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,610 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:02,610 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T12:55:02,611 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T12:55:02,611 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T12:55:02,611 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T12:55:02,611 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T12:55:02,611 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T12:55:02,612 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761702612,5,FailOnTimeoutGroup] 2024-11-16T12:55:02,612 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761702612,5,FailOnTimeoutGroup] 2024-11-16T12:55:02,612 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,612 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T12:55:02,612 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,612 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,623 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T12:55:02,623 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a 2024-11-16T12:55:02,628 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:55:02,629 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T12:55:02,631 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T12:55:02,631 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:02,632 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:55:02,632 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T12:55:02,635 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T12:55:02,635 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:02,635 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:55:02,635 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T12:55:02,638 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T12:55:02,638 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:02,638 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:55:02,639 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T12:55:02,641 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T12:55:02,641 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:02,642 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:55:02,642 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T12:55:02,642 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740 2024-11-16T12:55:02,642 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740 2024-11-16T12:55:02,643 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T12:55:02,643 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T12:55:02,643 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:55:02,643 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T12:55:02,648 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:55:02,648 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74235508, jitterRate=0.10619527101516724}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:55:02,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731761702628Initializing all the Stores at 1731761702628Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761702628Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761702628Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761702629 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761702629Cleaning up temporary data from old regions at 1731761702643 (+14 ms)Region opened successfully at 1731761702649 (+6 ms) 2024-11-16T12:55:02,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T12:55:02,649 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T12:55:02,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T12:55:02,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T12:55:02,650 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T12:55:02,650 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T12:55:02,650 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731761702649Disabling compacts and flushes for region at 1731761702649Disabling writes for close at 1731761702649Writing region close event to WAL at 1731761702650 (+1 ms)Closed at 1731761702650 2024-11-16T12:55:02,651 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:55:02,651 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T12:55:02,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T12:55:02,653 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T12:55:02,653 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T12:55:02,663 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer(746): ClusterId : 0ef62761-1acb-404b-8483-845645e00a64 2024-11-16T12:55:02,663 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.HRegionServer(746): ClusterId : 0ef62761-1acb-404b-8483-845645e00a64 2024-11-16T12:55:02,663 DEBUG [RS:1;0450ab8807f5:36895 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:55:02,663 DEBUG [RS:0;0450ab8807f5:41647 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:55:02,687 DEBUG [RS:0;0450ab8807f5:41647 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:55:02,687 DEBUG [RS:1;0450ab8807f5:36895 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:55:02,687 DEBUG [RS:1;0450ab8807f5:36895 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:55:02,687 DEBUG [RS:0;0450ab8807f5:41647 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:55:02,695 DEBUG [RS:1;0450ab8807f5:36895 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:55:02,695 DEBUG [RS:0;0450ab8807f5:41647 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:55:02,696 DEBUG [RS:1;0450ab8807f5:36895 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@291a9da6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:55:02,696 DEBUG [RS:0;0450ab8807f5:41647 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7364833a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:55:02,710 DEBUG [RS:0;0450ab8807f5:41647 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0450ab8807f5:41647 2024-11-16T12:55:02,710 DEBUG [RS:1;0450ab8807f5:36895 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0450ab8807f5:36895 2024-11-16T12:55:02,710 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:55:02,710 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:55:02,710 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:55:02,710 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:55:02,710 DEBUG [RS:0;0450ab8807f5:41647 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:55:02,710 DEBUG [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:55:02,711 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,38799,1731761702052 with port=41647, startcode=1731761702198 2024-11-16T12:55:02,711 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,38799,1731761702052 with port=36895, startcode=1731761702235 2024-11-16T12:55:02,711 DEBUG [RS:1;0450ab8807f5:36895 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:55:02,711 DEBUG [RS:0;0450ab8807f5:41647 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:55:02,714 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40619, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:55:02,714 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59927, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:55:02,714 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38799 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,36895,1731761702235 2024-11-16T12:55:02,714 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38799 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,36895,1731761702235 2024-11-16T12:55:02,715 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38799 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,41647,1731761702198 2024-11-16T12:55:02,716 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38799 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,41647,1731761702198 2024-11-16T12:55:02,716 DEBUG [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a 2024-11-16T12:55:02,716 DEBUG [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T12:55:02,716 DEBUG [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:55:02,717 DEBUG [RS:0;0450ab8807f5:41647 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a 2024-11-16T12:55:02,717 DEBUG [RS:0;0450ab8807f5:41647 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T12:55:02,717 DEBUG [RS:0;0450ab8807f5:41647 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:55:02,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:55:02,756 DEBUG [RS:1;0450ab8807f5:36895 {}] zookeeper.ZKUtil(111): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,36895,1731761702235 2024-11-16T12:55:02,757 WARN [RS:1;0450ab8807f5:36895 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:55:02,757 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,36895,1731761702235] 2024-11-16T12:55:02,757 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,41647,1731761702198] 2024-11-16T12:55:02,757 INFO [RS:1;0450ab8807f5:36895 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:55:02,757 DEBUG [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,36895,1731761702235 2024-11-16T12:55:02,757 DEBUG [RS:0;0450ab8807f5:41647 {}] zookeeper.ZKUtil(111): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,41647,1731761702198 2024-11-16T12:55:02,757 WARN [RS:0;0450ab8807f5:41647 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:55:02,757 INFO [RS:0;0450ab8807f5:41647 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:55:02,757 DEBUG [RS:0;0450ab8807f5:41647 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,41647,1731761702198 2024-11-16T12:55:02,763 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:55:02,763 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:55:02,765 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:55:02,766 INFO [RS:0;0450ab8807f5:41647 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:55:02,766 INFO [RS:0;0450ab8807f5:41647 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,766 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:55:02,767 INFO [RS:0;0450ab8807f5:41647 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:55:02,767 INFO [RS:0;0450ab8807f5:41647 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,767 DEBUG [RS:0;0450ab8807f5:41647 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,768 DEBUG [RS:0;0450ab8807f5:41647 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,768 DEBUG [RS:0;0450ab8807f5:41647 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,768 DEBUG [RS:0;0450ab8807f5:41647 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,768 DEBUG [RS:0;0450ab8807f5:41647 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,768 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:55:02,768 DEBUG [RS:0;0450ab8807f5:41647 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:55:02,768 DEBUG [RS:0;0450ab8807f5:41647 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,768 DEBUG [RS:0;0450ab8807f5:41647 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,769 INFO [RS:1;0450ab8807f5:36895 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:55:02,769 DEBUG [RS:0;0450ab8807f5:41647 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,769 INFO [RS:1;0450ab8807f5:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,769 DEBUG [RS:0;0450ab8807f5:41647 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,769 DEBUG [RS:0;0450ab8807f5:41647 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,769 DEBUG [RS:0;0450ab8807f5:41647 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,769 DEBUG [RS:0;0450ab8807f5:41647 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:55:02,769 DEBUG [RS:0;0450ab8807f5:41647 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:55:02,769 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:55:02,770 INFO [RS:1;0450ab8807f5:36895 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:55:02,770 INFO [RS:0;0450ab8807f5:41647 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,770 INFO [RS:1;0450ab8807f5:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,771 INFO [RS:0;0450ab8807f5:41647 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,771 INFO [RS:0;0450ab8807f5:41647 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,771 DEBUG [RS:1;0450ab8807f5:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,771 INFO [RS:0;0450ab8807f5:41647 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,771 INFO [RS:0;0450ab8807f5:41647 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,771 DEBUG [RS:1;0450ab8807f5:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,771 INFO [RS:0;0450ab8807f5:41647 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,41647,1731761702198-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:55:02,771 DEBUG [RS:1;0450ab8807f5:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,771 DEBUG [RS:1;0450ab8807f5:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,771 DEBUG [RS:1;0450ab8807f5:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,772 DEBUG [RS:1;0450ab8807f5:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:55:02,772 DEBUG [RS:1;0450ab8807f5:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,772 DEBUG [RS:1;0450ab8807f5:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,772 DEBUG [RS:1;0450ab8807f5:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,772 DEBUG [RS:1;0450ab8807f5:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,772 DEBUG [RS:1;0450ab8807f5:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,773 DEBUG [RS:1;0450ab8807f5:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:02,773 DEBUG [RS:1;0450ab8807f5:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:55:02,773 DEBUG [RS:1;0450ab8807f5:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:55:02,774 INFO [RS:1;0450ab8807f5:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,774 INFO [RS:1;0450ab8807f5:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,774 INFO [RS:1;0450ab8807f5:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,774 INFO [RS:1;0450ab8807f5:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,774 INFO [RS:1;0450ab8807f5:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,775 INFO [RS:1;0450ab8807f5:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,36895,1731761702235-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:55:02,786 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:55:02,786 INFO [RS:0;0450ab8807f5:41647 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,41647,1731761702198-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,787 INFO [RS:0;0450ab8807f5:41647 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,787 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.Replication(171): 0450ab8807f5,41647,1731761702198 started 2024-11-16T12:55:02,789 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:55:02,789 INFO [RS:1;0450ab8807f5:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,36895,1731761702235-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,789 INFO [RS:1;0450ab8807f5:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,789 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.Replication(171): 0450ab8807f5,36895,1731761702235 started 2024-11-16T12:55:02,799 INFO [RS:0;0450ab8807f5:41647 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,800 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,41647,1731761702198, RpcServer on 0450ab8807f5/172.17.0.2:41647, sessionid=0x1014500175b0001 2024-11-16T12:55:02,800 DEBUG [RS:0;0450ab8807f5:41647 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:55:02,800 DEBUG [RS:0;0450ab8807f5:41647 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,41647,1731761702198 2024-11-16T12:55:02,800 DEBUG [RS:0;0450ab8807f5:41647 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,41647,1731761702198' 2024-11-16T12:55:02,800 DEBUG [RS:0;0450ab8807f5:41647 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:55:02,801 DEBUG [RS:0;0450ab8807f5:41647 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:55:02,801 DEBUG [RS:0;0450ab8807f5:41647 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:55:02,801 DEBUG [RS:0;0450ab8807f5:41647 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:55:02,801 DEBUG [RS:0;0450ab8807f5:41647 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,41647,1731761702198 2024-11-16T12:55:02,801 DEBUG [RS:0;0450ab8807f5:41647 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,41647,1731761702198' 2024-11-16T12:55:02,801 DEBUG [RS:0;0450ab8807f5:41647 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:55:02,802 DEBUG [RS:0;0450ab8807f5:41647 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:55:02,802 DEBUG [RS:0;0450ab8807f5:41647 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:55:02,802 INFO [RS:1;0450ab8807f5:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:02,802 INFO [RS:0;0450ab8807f5:41647 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:55:02,802 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,36895,1731761702235, RpcServer on 0450ab8807f5/172.17.0.2:36895, sessionid=0x1014500175b0002 2024-11-16T12:55:02,802 INFO [RS:0;0450ab8807f5:41647 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:55:02,802 DEBUG [RS:1;0450ab8807f5:36895 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:55:02,802 DEBUG [RS:1;0450ab8807f5:36895 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,36895,1731761702235 2024-11-16T12:55:02,803 DEBUG [RS:1;0450ab8807f5:36895 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,36895,1731761702235' 2024-11-16T12:55:02,803 DEBUG [RS:1;0450ab8807f5:36895 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:55:02,803 DEBUG [RS:1;0450ab8807f5:36895 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:55:02,804 WARN [0450ab8807f5:38799 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-16T12:55:02,804 DEBUG [RS:1;0450ab8807f5:36895 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:55:02,804 DEBUG [RS:1;0450ab8807f5:36895 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:55:02,804 DEBUG [RS:1;0450ab8807f5:36895 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,36895,1731761702235 2024-11-16T12:55:02,804 DEBUG [RS:1;0450ab8807f5:36895 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,36895,1731761702235' 2024-11-16T12:55:02,804 DEBUG [RS:1;0450ab8807f5:36895 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:55:02,804 DEBUG [RS:1;0450ab8807f5:36895 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:55:02,805 DEBUG [RS:1;0450ab8807f5:36895 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:55:02,805 INFO [RS:1;0450ab8807f5:36895 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:55:02,805 INFO [RS:1;0450ab8807f5:36895 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:55:02,903 INFO [RS:0;0450ab8807f5:41647 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:55:02,904 INFO [RS:0;0450ab8807f5:41647 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0450ab8807f5%2C41647%2C1731761702198, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,41647,1731761702198, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/oldWALs, maxLogs=55 2024-11-16T12:55:02,905 INFO [RS:1;0450ab8807f5:36895 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:55:02,906 INFO [RS:1;0450ab8807f5:36895 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0450ab8807f5%2C36895%2C1731761702235, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,36895,1731761702235, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/oldWALs, maxLogs=55 2024-11-16T12:55:02,923 INFO [RS:0;0450ab8807f5:41647 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,41647,1731761702198/0450ab8807f5%2C41647%2C1731761702198.1731761702905 2024-11-16T12:55:02,923 DEBUG [RS:0;0450ab8807f5:41647 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T12:55:02,923 INFO [RS:1;0450ab8807f5:36895 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,36895,1731761702235/0450ab8807f5%2C36895%2C1731761702235.1731761702907 2024-11-16T12:55:02,924 DEBUG [RS:1;0450ab8807f5:36895 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T12:55:03,054 DEBUG [0450ab8807f5:38799 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-16T12:55:03,055 DEBUG [0450ab8807f5:38799 {}] balancer.BalancerClusterState(204): Hosts are {0450ab8807f5=0} racks are {/default-rack=0} 2024-11-16T12:55:03,057 DEBUG [0450ab8807f5:38799 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T12:55:03,057 DEBUG [0450ab8807f5:38799 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T12:55:03,057 DEBUG [0450ab8807f5:38799 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T12:55:03,057 DEBUG [0450ab8807f5:38799 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T12:55:03,057 INFO [0450ab8807f5:38799 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T12:55:03,057 INFO [0450ab8807f5:38799 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T12:55:03,057 DEBUG [0450ab8807f5:38799 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T12:55:03,058 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0450ab8807f5,36895,1731761702235 2024-11-16T12:55:03,061 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,36895,1731761702235, state=OPENING 2024-11-16T12:55:03,086 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T12:55:03,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:03,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:03,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:03,095 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T12:55:03,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0450ab8807f5,36895,1731761702235}] 2024-11-16T12:55:03,095 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:55:03,095 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:55:03,095 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:55:03,248 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:55:03,251 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50107, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:55:03,254 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T12:55:03,254 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:55:03,255 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T12:55:03,257 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0450ab8807f5%2C36895%2C1731761702235.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,36895,1731761702235, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/oldWALs, maxLogs=55 2024-11-16T12:55:03,274 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,36895,1731761702235/0450ab8807f5%2C36895%2C1731761702235.meta.1731761703258.meta 2024-11-16T12:55:03,274 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T12:55:03,274 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:55:03,275 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T12:55:03,275 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T12:55:03,275 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T12:55:03,275 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T12:55:03,275 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:55:03,276 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T12:55:03,276 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T12:55:03,277 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T12:55:03,277 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T12:55:03,277 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:03,278 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:55:03,278 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T12:55:03,278 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T12:55:03,278 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:03,278 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:55:03,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T12:55:03,279 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T12:55:03,279 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:03,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:55:03,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T12:55:03,280 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T12:55:03,280 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:03,280 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T12:55:03,280 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T12:55:03,281 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740 2024-11-16T12:55:03,281 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740 2024-11-16T12:55:03,281 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T12:55:03,281 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T12:55:03,281 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T12:55:03,282 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T12:55:03,283 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67166425, jitterRate=8.577257394790649E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T12:55:03,283 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T12:55:03,283 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731761703276Writing region info on filesystem at 1731761703276Initializing all the Stores at 1731761703276Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761703276Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761703277 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761703277Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731761703277Cleaning up temporary data from old regions at 1731761703281 (+4 ms)Running coprocessor post-open hooks at 1731761703283 (+2 ms)Region opened successfully at 1731761703283 2024-11-16T12:55:03,284 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731761703248 2024-11-16T12:55:03,287 DEBUG [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T12:55:03,287 INFO [RS_OPEN_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T12:55:03,287 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,36895,1731761702235 2024-11-16T12:55:03,288 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0450ab8807f5,36895,1731761702235, state=OPEN 2024-11-16T12:55:03,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:55:03,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:55:03,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T12:55:03,294 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:55:03,294 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:55:03,294 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T12:55:03,294 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0450ab8807f5,36895,1731761702235 2024-11-16T12:55:03,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T12:55:03,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0450ab8807f5,36895,1731761702235 in 199 msec 2024-11-16T12:55:03,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T12:55:03,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 645 msec 2024-11-16T12:55:03,298 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T12:55:03,298 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T12:55:03,301 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:55:03,301 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,36895,1731761702235, seqNum=-1] 2024-11-16T12:55:03,301 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:55:03,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58337, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:55:03,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 702 msec 2024-11-16T12:55:03,309 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731761703309, completionTime=-1 2024-11-16T12:55:03,309 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-16T12:55:03,310 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T12:55:03,312 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T12:55:03,312 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731761763312 2024-11-16T12:55:03,312 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731761823312 2024-11-16T12:55:03,312 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-16T12:55:03,313 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,38799,1731761702052-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:03,313 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,38799,1731761702052-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:03,313 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,38799,1731761702052-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:03,313 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0450ab8807f5:38799, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:03,313 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:03,314 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:03,317 DEBUG [master/0450ab8807f5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T12:55:03,320 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.025sec 2024-11-16T12:55:03,320 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T12:55:03,320 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T12:55:03,320 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T12:55:03,320 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T12:55:03,320 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T12:55:03,321 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,38799,1731761702052-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:55:03,321 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,38799,1731761702052-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T12:55:03,322 DEBUG [master/0450ab8807f5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T12:55:03,322 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T12:55:03,322 INFO [master/0450ab8807f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,38799,1731761702052-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:03,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T12:55:03,363 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0450ab8807f5,38799,-1 for getting cluster id 2024-11-16T12:55:03,363 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T12:55:03,365 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0ef62761-1acb-404b-8483-845645e00a64' 2024-11-16T12:55:03,366 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T12:55:03,366 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0ef62761-1acb-404b-8483-845645e00a64" 2024-11-16T12:55:03,366 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34667042, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T12:55:03,366 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0450ab8807f5,38799,-1] 2024-11-16T12:55:03,366 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T12:55:03,367 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:03,369 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T12:55:03,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fb9de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T12:55:03,370 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T12:55:03,372 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0450ab8807f5,36895,1731761702235, seqNum=-1] 2024-11-16T12:55:03,372 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:55:03,374 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:55:03,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0450ab8807f5,38799,1731761702052 2024-11-16T12:55:03,377 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T12:55:03,378 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 0450ab8807f5,38799,1731761702052 2024-11-16T12:55:03,378 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1eb3b318 2024-11-16T12:55:03,379 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T12:55:03,381 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T12:55:03,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38799 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731761703376', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T12:55:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38799 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731761703376 2024-11-16T12:55:03,385 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731761703376 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T12:55:03,385 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:03,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38799 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731761703376" procId is: 4 2024-11-16T12:55:03,386 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731761703376 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T12:55:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:55:03,394 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731761703376-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => df427f7c2a516b6a4dc66ac07c6d145d, NAME => 'testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731761703376', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a 2024-11-16T12:55:03,399 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731761703376-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:55:03,400 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731761703376-pool-0 {}] regionserver.HRegion(1722): Closing df427f7c2a516b6a4dc66ac07c6d145d, disabling compactions & flushes 2024-11-16T12:55:03,400 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731761703376-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:03,400 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731761703376-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:03,400 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731761703376-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. after waiting 0 ms 2024-11-16T12:55:03,400 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731761703376-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:03,400 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731761703376-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:03,400 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731761703376-pool-0 {}] regionserver.HRegion(1676): Region close journal for df427f7c2a516b6a4dc66ac07c6d145d: Waiting for close lock at 1731761703399Disabling compacts and flushes for region at 1731761703399Disabling writes for close at 1731761703400 (+1 ms)Writing region close event to WAL at 1731761703400Closed at 1731761703400 2024-11-16T12:55:03,401 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731761703376 execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T12:55:03,401 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731761703401"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731761703401"}]},"ts":"1731761703401"} 2024-11-16T12:55:03,404 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T12:55:03,405 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731761703376 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T12:55:03,405 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731761703376","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731761703405"}]},"ts":"1731761703405"} 2024-11-16T12:55:03,407 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731761703376, state=ENABLING in hbase:meta 2024-11-16T12:55:03,407 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {0450ab8807f5=0} racks are {/default-rack=0} 2024-11-16T12:55:03,408 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T12:55:03,408 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T12:55:03,408 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T12:55:03,408 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T12:55:03,408 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T12:55:03,408 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T12:55:03,408 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T12:55:03,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731761703376, region=df427f7c2a516b6a4dc66ac07c6d145d, ASSIGN}] 2024-11-16T12:55:03,409 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731761703376, region=df427f7c2a516b6a4dc66ac07c6d145d, ASSIGN 2024-11-16T12:55:03,410 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731761703376, region=df427f7c2a516b6a4dc66ac07c6d145d, ASSIGN; state=OFFLINE, location=0450ab8807f5,41647,1731761702198; forceNewPlan=false, retain=false 2024-11-16T12:55:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:55:03,561 INFO [0450ab8807f5:38799 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T12:55:03,562 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=df427f7c2a516b6a4dc66ac07c6d145d, regionState=OPENING, regionLocation=0450ab8807f5,41647,1731761702198 2024-11-16T12:55:03,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731761703376, region=df427f7c2a516b6a4dc66ac07c6d145d, ASSIGN because future has completed 2024-11-16T12:55:03,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure df427f7c2a516b6a4dc66ac07c6d145d, server=0450ab8807f5,41647,1731761702198}] 2024-11-16T12:55:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:55:03,724 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T12:55:03,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40167, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T12:55:03,732 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:03,732 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => df427f7c2a516b6a4dc66ac07c6d145d, NAME => 'testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:55:03,733 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731761703376 df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:03,733 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:55:03,733 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:03,733 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:03,734 INFO [StoreOpener-df427f7c2a516b6a4dc66ac07c6d145d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:03,738 INFO [StoreOpener-df427f7c2a516b6a4dc66ac07c6d145d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df427f7c2a516b6a4dc66ac07c6d145d columnFamilyName family 2024-11-16T12:55:03,738 DEBUG [StoreOpener-df427f7c2a516b6a4dc66ac07c6d145d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:03,738 INFO [StoreOpener-df427f7c2a516b6a4dc66ac07c6d145d-1 {}] regionserver.HStore(327): Store=df427f7c2a516b6a4dc66ac07c6d145d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:55:03,738 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:03,739 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:03,739 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:03,739 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:03,739 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:03,740 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:03,747 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T12:55:03,747 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened df427f7c2a516b6a4dc66ac07c6d145d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63760414, jitterRate=-0.049895793199539185}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:55:03,747 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:03,748 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for df427f7c2a516b6a4dc66ac07c6d145d: Running coprocessor pre-open hook at 1731761703733Writing region info on filesystem at 1731761703733Initializing all the Stores at 1731761703733Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761703733Cleaning up temporary data from old regions at 1731761703740 (+7 ms)Running coprocessor post-open hooks at 1731761703747 (+7 ms)Region opened successfully at 1731761703748 (+1 ms) 2024-11-16T12:55:03,749 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d., pid=6, masterSystemTime=1731761703723 2024-11-16T12:55:03,752 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:03,752 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:03,753 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=df427f7c2a516b6a4dc66ac07c6d145d, regionState=OPEN, openSeqNum=2, regionLocation=0450ab8807f5,41647,1731761702198 2024-11-16T12:55:03,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure df427f7c2a516b6a4dc66ac07c6d145d, server=0450ab8807f5,41647,1731761702198 because future has completed 2024-11-16T12:55:03,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T12:55:03,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure df427f7c2a516b6a4dc66ac07c6d145d, server=0450ab8807f5,41647,1731761702198 in 190 msec 2024-11-16T12:55:03,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T12:55:03,763 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731761703376, region=df427f7c2a516b6a4dc66ac07c6d145d, ASSIGN in 352 msec 2024-11-16T12:55:03,763 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731761703376 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T12:55:03,764 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731761703376","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731761703764"}]},"ts":"1731761703764"} 2024-11-16T12:55:03,766 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731761703376, state=ENABLED in hbase:meta 2024-11-16T12:55:03,767 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731761703376 execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T12:55:03,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731761703376 in 385 msec 2024-11-16T12:55:04,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38799 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T12:55:04,014 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731761703376 completed 2024-11-16T12:55:04,014 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731761703376 get assigned. Timeout = 60000ms 2024-11-16T12:55:04,015 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T12:55:04,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731761703376 assigned to meta. Checking AM states. 2024-11-16T12:55:04,020 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T12:55:04,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731761703376 assigned. 2024-11-16T12:55:04,025 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731761703376', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d., hostname=0450ab8807f5,41647,1731761702198, seqNum=2] 2024-11-16T12:55:04,025 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T12:55:04,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T12:55:04,033 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing df427f7c2a516b6a4dc66ac07c6d145d 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-16T12:55:04,040 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 496bb1911f23400a8b95077d4757d038 2024-11-16T12:55:04,054 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/.tmp/family/496bb1911f23400a8b95077d4757d038 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-16T12:55:04,055 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/.tmp/family/496bb1911f23400a8b95077d4757d038 2024-11-16T12:55:04,057 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/.tmp/family/496bb1911f23400a8b95077d4757d038 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/family/496bb1911f23400a8b95077d4757d038 2024-11-16T12:55:04,058 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/family/496bb1911f23400a8b95077d4757d038, delay=907 ms 2024-11-16T12:55:04,059 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/family/496bb1911f23400a8b95077d4757d038, entries=2, sequenceid=6, filesize=4.9 K 2024-11-16T12:55:04,060 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for df427f7c2a516b6a4dc66ac07c6d145d in 27ms, sequenceid=6, compaction requested=false 2024-11-16T12:55:04,060 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731761703376' 2024-11-16T12:55:04,060 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for df427f7c2a516b6a4dc66ac07c6d145d: 2024-11-16T12:55:04,313 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T12:55:04,966 DEBUG [hfile-prefetch-1731761701451 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 496bb1911f23400a8b95077d4757d038_0. 2024-11-16T12:55:04,967 DEBUG [hfile-prefetch-1731761701451 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 496bb1911f23400a8b95077d4757d038_117. 2024-11-16T12:55:04,968 DEBUG [hfile-prefetch-1731761701451 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/family/496bb1911f23400a8b95077d4757d038, with total blocks 2, and data blocks 1 2024-11-16T12:55:04,968 DEBUG [hfile-prefetch-1731761701451 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-16T12:55:04,968 DEBUG [hfile-prefetch-1731761701451 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/family/496bb1911f23400a8b95077d4757d038, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-16T12:55:04,968 DEBUG [hfile-prefetch-1731761701451 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/family/496bb1911f23400a8b95077d4757d038. Will sleep 100ms and try the verification again. 2024-11-16T12:55:05,055 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-16T12:55:05,055 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-16T12:55:05,056 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T12:55:05,056 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T12:55:05,058 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T12:55:05,058 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731761705055 2024-11-16T12:55:05,059 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-16T12:55:05,061 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;0450ab8807f5:41647,5,FailOnTimeoutGroup] 2024-11-16T12:55:05,061 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,41647,1731761702198' ***** 2024-11-16T12:55:05,061 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-16T12:55:05,061 INFO [RS:0;0450ab8807f5:41647 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T12:55:05,061 INFO [RS:0;0450ab8807f5:41647 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/bucket.persistence 2024-11-16T12:55:05,061 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T12:55:05,061 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T12:55:05,061 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T12:55:05,061 INFO [RS:0;0450ab8807f5:41647 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T12:55:05,061 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T12:55:05,068 DEBUG [hfile-prefetch-1731761701451 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/family/496bb1911f23400a8b95077d4757d038, with total blocks 2, and data blocks 1 2024-11-16T12:55:05,068 DEBUG [hfile-prefetch-1731761701451 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-16T12:55:05,068 DEBUG [hfile-prefetch-1731761701451 {}] bucket.BucketCache(2293): found block for file 496bb1911f23400a8b95077d4757d038 in the backing map. Acquiring read lock for offset 0 2024-11-16T12:55:05,069 DEBUG [hfile-prefetch-1731761701451 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/family/496bb1911f23400a8b95077d4757d038 has now been fully cached. 2024-11-16T12:55:05,069 DEBUG [hfile-prefetch-1731761701451 {}] hfile.PrefetchExecutor(121): Prefetch completed for 496bb1911f23400a8b95077d4757d038 2024-11-16T12:55:05,072 DEBUG [RS:0;0450ab8807f5:41647 {}] bucket.BucketCache(1487): Thread RS:0;0450ab8807f5:41647 started persisting bucket cache to file 2024-11-16T12:55:05,072 DEBUG [RS:0;0450ab8807f5:41647 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T12:55:05,072 DEBUG [RS:0;0450ab8807f5:41647 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-16T12:55:05,072 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:55:05,073 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:55:05,075 DEBUG [RS:0;0450ab8807f5:41647 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-16T12:55:05,075 DEBUG [RS:0;0450ab8807f5:41647 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731761705072 2024-11-16T12:55:05,075 DEBUG [RS:0;0450ab8807f5:41647 {}] bucket.BucketCache(1509): Thread RS:0;0450ab8807f5:41647 finished persisting bucket cache to file, renaming 2024-11-16T12:55:05,075 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:55:05,075 INFO [RS:0;0450ab8807f5:41647 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T12:55:05,076 INFO [RS:0;0450ab8807f5:41647 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T12:55:05,076 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.HRegionServer(3091): Received CLOSE for df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:05,076 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.HRegionServer(959): stopping server 0450ab8807f5,41647,1731761702198 2024-11-16T12:55:05,076 INFO [RS:0;0450ab8807f5:41647 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:55:05,076 INFO [RS:0;0450ab8807f5:41647 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0450ab8807f5:41647. 2024-11-16T12:55:05,076 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T12:55:05,076 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing df427f7c2a516b6a4dc66ac07c6d145d, disabling compactions & flushes 2024-11-16T12:55:05,076 DEBUG [RS:0;0450ab8807f5:41647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:55:05,076 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:05,076 DEBUG [RS:0;0450ab8807f5:41647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:05,076 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:05,076 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. after waiting 0 ms 2024-11-16T12:55:05,076 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:05,077 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T12:55:05,077 DEBUG [RS:0;0450ab8807f5:41647 {}] regionserver.HRegionServer(1325): Online Regions={df427f7c2a516b6a4dc66ac07c6d145d=testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d.} 2024-11-16T12:55:05,077 DEBUG [RS:0;0450ab8807f5:41647 {}] regionserver.HRegionServer(1351): Waiting on df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:05,081 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-16T12:55:05,082 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:05,082 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for df427f7c2a516b6a4dc66ac07c6d145d: Waiting for close lock at 1731761705076Running coprocessor pre-close hooks at 1731761705076Disabling compacts and flushes for region at 1731761705076Disabling writes for close at 1731761705076Writing region close event to WAL at 1731761705077 (+1 ms)Running coprocessor post-close hooks at 1731761705082 (+5 ms)Closed at 1731761705082 2024-11-16T12:55:05,082 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:05,277 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.HRegionServer(976): stopping server 0450ab8807f5,41647,1731761702198; all regions closed. 2024-11-16T12:55:05,280 DEBUG [RS:0;0450ab8807f5:41647 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/oldWALs 2024-11-16T12:55:05,280 INFO [RS:0;0450ab8807f5:41647 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0450ab8807f5%2C41647%2C1731761702198:(num 1731761702905) 2024-11-16T12:55:05,280 DEBUG [RS:0;0450ab8807f5:41647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:05,280 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:55:05,280 INFO [RS:0;0450ab8807f5:41647 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:55:05,280 INFO [RS:0;0450ab8807f5:41647 {}] hbase.ChoreService(370): Chore service for: regionserver/0450ab8807f5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T12:55:05,280 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T12:55:05,280 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T12:55:05,280 INFO [regionserver/0450ab8807f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:55:05,281 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T12:55:05,281 INFO [RS:0;0450ab8807f5:41647 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:55:05,281 INFO [RS:0;0450ab8807f5:41647 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41647 2024-11-16T12:55:05,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:55:05,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0450ab8807f5,41647,1731761702198 2024-11-16T12:55:05,319 INFO [RS:0;0450ab8807f5:41647 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:55:05,327 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0450ab8807f5,41647,1731761702198] 2024-11-16T12:55:05,335 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0450ab8807f5,41647,1731761702198 already deleted, retry=false 2024-11-16T12:55:05,335 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 0450ab8807f5,41647,1731761702198 on 0450ab8807f5,38799,1731761702052 2024-11-16T12:55:05,339 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 0450ab8807f5,41647,1731761702198, splitWal=true, meta=false 2024-11-16T12:55:05,341 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 0450ab8807f5,41647,1731761702198, splitWal=true, meta=false 2024-11-16T12:55:05,341 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 0450ab8807f5,41647,1731761702198 (carryingMeta=false) 0450ab8807f5,41647,1731761702198/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1c525ef4[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-16T12:55:05,342 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 0450ab8807f5,41647,1731761702198 had 1 regions 2024-11-16T12:55:05,343 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 0450ab8807f5,41647,1731761702198, splitWal=true, meta=false, isMeta: false 2024-11-16T12:55:05,343 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,41647,1731761702198-splitting 2024-11-16T12:55:05,344 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,41647,1731761702198-splitting dir is empty, no logs to split. 2024-11-16T12:55:05,344 INFO [PEWorker-2 {}] master.SplitWALManager(105): 0450ab8807f5,41647,1731761702198 WAL count=0, meta=false 2024-11-16T12:55:05,345 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,41647,1731761702198-splitting dir is empty, no logs to split. 2024-11-16T12:55:05,345 INFO [PEWorker-2 {}] master.SplitWALManager(105): 0450ab8807f5,41647,1731761702198 WAL count=0, meta=false 2024-11-16T12:55:05,345 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 0450ab8807f5,41647,1731761702198 WAL splitting is done? wals=0, meta=false 2024-11-16T12:55:05,346 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 0450ab8807f5,41647,1731761702198 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,41647,1731761702198-splitting does not exist 2024-11-16T12:55:05,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731761703376, region=df427f7c2a516b6a4dc66ac07c6d145d, ASSIGN}] 2024-11-16T12:55:05,348 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731761703376, region=df427f7c2a516b6a4dc66ac07c6d145d, ASSIGN 2024-11-16T12:55:05,349 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731761703376, region=df427f7c2a516b6a4dc66ac07c6d145d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T12:55:05,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:55:05,427 INFO [RS:0;0450ab8807f5:41647 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:55:05,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41647-0x1014500175b0001, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:55:05,427 INFO [RS:0;0450ab8807f5:41647 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,41647,1731761702198; zookeeper connection closed. 2024-11-16T12:55:05,428 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@612ac04 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@612ac04 2024-11-16T12:55:05,500 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=df427f7c2a516b6a4dc66ac07c6d145d, regionState=OPENING, regionLocation=0450ab8807f5,36895,1731761702235 2024-11-16T12:55:05,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731761703376, region=df427f7c2a516b6a4dc66ac07c6d145d, ASSIGN because future has completed 2024-11-16T12:55:05,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure df427f7c2a516b6a4dc66ac07c6d145d, server=0450ab8807f5,36895,1731761702235}] 2024-11-16T12:55:05,577 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0450ab8807f5:0 server-side Connection retries=45 2024-11-16T12:55:05,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:55:05,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T12:55:05,577 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T12:55:05,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T12:55:05,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T12:55:05,577 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T12:55:05,578 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T12:55:05,578 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46513 2024-11-16T12:55:05,580 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46513 connecting to ZooKeeper ensemble=127.0.0.1:53805 2024-11-16T12:55:05,580 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:55:05,580 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:55:05,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:465130x0, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T12:55:05,611 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46513-0x1014500175b0003 connected 2024-11-16T12:55:05,611 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46513-0x1014500175b0003, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-16T12:55:05,611 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-16T12:55:05,613 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T12:55:05,613 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/bucket.cache 2024-11-16T12:55:05,613 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T12:55:05,614 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T12:55:05,614 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T12:55:05,614 INFO [Thread-201 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T12:55:05,614 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T12:55:05,615 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T12:55:05,615 INFO [Thread-201 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-16T12:55:05,616 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46513-0x1014500175b0003, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T12:55:05,617 INFO [Thread-201 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-16T12:55:05,619 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46513-0x1014500175b0003, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T12:55:05,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46513 2024-11-16T12:55:05,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46513 2024-11-16T12:55:05,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46513 2024-11-16T12:55:05,623 DEBUG [Thread-201 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-16T12:55:05,623 DEBUG [Thread-201 {}] bucket.BucketCache(1584): Cached File Entry:<496bb1911f23400a8b95077d4757d038,> 2024-11-16T12:55:05,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46513 2024-11-16T12:55:05,624 INFO [Thread-201 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T12:55:05,625 INFO [Thread-201 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-16T12:55:05,625 INFO [Thread-201 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/bucket.persistence is complete. 2024-11-16T12:55:05,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46513 2024-11-16T12:55:05,628 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.HRegionServer(746): ClusterId : 0ef62761-1acb-404b-8483-845645e00a64 2024-11-16T12:55:05,628 DEBUG [RS:2;0450ab8807f5:46513 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T12:55:05,636 DEBUG [RS:2;0450ab8807f5:46513 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T12:55:05,636 DEBUG [RS:2;0450ab8807f5:46513 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T12:55:05,644 DEBUG [RS:2;0450ab8807f5:46513 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T12:55:05,645 DEBUG [RS:2;0450ab8807f5:46513 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eb6266b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0450ab8807f5/172.17.0.2:0 2024-11-16T12:55:05,656 DEBUG [RS:2;0450ab8807f5:46513 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;0450ab8807f5:46513 2024-11-16T12:55:05,656 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T12:55:05,656 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T12:55:05,656 DEBUG [RS:2;0450ab8807f5:46513 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T12:55:05,657 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.HRegionServer(2659): reportForDuty to master=0450ab8807f5,38799,1731761702052 with port=46513, startcode=1731761705576 2024-11-16T12:55:05,657 DEBUG [RS:2;0450ab8807f5:46513 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T12:55:05,659 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48169, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T12:55:05,659 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38799 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0450ab8807f5,46513,1731761705576 2024-11-16T12:55:05,660 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38799 {}] master.ServerManager(517): Registering regionserver=0450ab8807f5,46513,1731761705576 2024-11-16T12:55:05,661 DEBUG [RS:2;0450ab8807f5:46513 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a 2024-11-16T12:55:05,661 DEBUG [RS:2;0450ab8807f5:46513 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T12:55:05,661 DEBUG [RS:2;0450ab8807f5:46513 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T12:55:05,662 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:05,663 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => df427f7c2a516b6a4dc66ac07c6d145d, NAME => 'testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d.', STARTKEY => '', ENDKEY => ''} 2024-11-16T12:55:05,663 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731761703376 df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:05,663 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T12:55:05,663 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:05,663 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:05,664 INFO [StoreOpener-df427f7c2a516b6a4dc66ac07c6d145d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:05,664 INFO [StoreOpener-df427f7c2a516b6a4dc66ac07c6d145d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df427f7c2a516b6a4dc66ac07c6d145d columnFamilyName family 2024-11-16T12:55:05,664 DEBUG [StoreOpener-df427f7c2a516b6a4dc66ac07c6d145d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T12:55:05,666 DEBUG [StoreFileOpener-df427f7c2a516b6a4dc66ac07c6d145d-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/family/496bb1911f23400a8b95077d4757d038, delay=938 ms 2024-11-16T12:55:05,667 DEBUG [StoreOpener-df427f7c2a516b6a4dc66ac07c6d145d-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/family/496bb1911f23400a8b95077d4757d038 2024-11-16T12:55:05,667 INFO [StoreOpener-df427f7c2a516b6a4dc66ac07c6d145d-1 {}] regionserver.HStore(327): Store=df427f7c2a516b6a4dc66ac07c6d145d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T12:55:05,667 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:05,668 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:05,668 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:05,668 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:05,668 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:05,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:55:05,669 DEBUG [RS:2;0450ab8807f5:46513 {}] zookeeper.ZKUtil(111): regionserver:46513-0x1014500175b0003, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0450ab8807f5,46513,1731761705576 2024-11-16T12:55:05,669 WARN [RS:2;0450ab8807f5:46513 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T12:55:05,669 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0450ab8807f5,46513,1731761705576] 2024-11-16T12:55:05,669 INFO [RS:2;0450ab8807f5:46513 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T12:55:05,669 DEBUG [RS:2;0450ab8807f5:46513 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,46513,1731761705576 2024-11-16T12:55:05,669 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:05,670 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened df427f7c2a516b6a4dc66ac07c6d145d; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75016876, jitterRate=0.11783856153488159}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T12:55:05,670 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:05,671 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for df427f7c2a516b6a4dc66ac07c6d145d: Running coprocessor pre-open hook at 1731761705663Writing region info on filesystem at 1731761705663Initializing all the Stores at 1731761705663Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731761705663Cleaning up temporary data from old regions at 1731761705668 (+5 ms)Running coprocessor post-open hooks at 1731761705670 (+2 ms)Region opened successfully at 1731761705671 (+1 ms) 2024-11-16T12:55:05,671 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d., pid=9, masterSystemTime=1731761705660 2024-11-16T12:55:05,674 DEBUG [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:05,674 INFO [RS_OPEN_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:05,674 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=df427f7c2a516b6a4dc66ac07c6d145d, regionState=OPEN, openSeqNum=10, regionLocation=0450ab8807f5,36895,1731761702235 2024-11-16T12:55:05,674 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T12:55:05,677 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T12:55:05,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure df427f7c2a516b6a4dc66ac07c6d145d, server=0450ab8807f5,36895,1731761702235 because future has completed 2024-11-16T12:55:05,677 INFO [RS:2;0450ab8807f5:46513 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T12:55:05,677 INFO [RS:2;0450ab8807f5:46513 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:05,678 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T12:55:05,679 INFO [RS:2;0450ab8807f5:46513 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T12:55:05,679 INFO [RS:2;0450ab8807f5:46513 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:05,679 DEBUG [RS:2;0450ab8807f5:46513 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:05,679 DEBUG [RS:2;0450ab8807f5:46513 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:05,680 DEBUG [RS:2;0450ab8807f5:46513 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:05,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-16T12:55:05,680 DEBUG [RS:2;0450ab8807f5:46513 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:05,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure df427f7c2a516b6a4dc66ac07c6d145d, server=0450ab8807f5,36895,1731761702235 in 171 msec 2024-11-16T12:55:05,680 DEBUG [RS:2;0450ab8807f5:46513 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:05,680 DEBUG [RS:2;0450ab8807f5:46513 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0450ab8807f5:0, corePoolSize=2, maxPoolSize=2 2024-11-16T12:55:05,680 DEBUG [RS:2;0450ab8807f5:46513 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:05,680 DEBUG [RS:2;0450ab8807f5:46513 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:05,680 DEBUG [RS:2;0450ab8807f5:46513 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:05,680 DEBUG [RS:2;0450ab8807f5:46513 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:05,680 DEBUG [RS:2;0450ab8807f5:46513 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:05,680 DEBUG [RS:2;0450ab8807f5:46513 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0450ab8807f5:0, corePoolSize=1, maxPoolSize=1 2024-11-16T12:55:05,680 DEBUG [RS:2;0450ab8807f5:46513 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:55:05,680 DEBUG [RS:2;0450ab8807f5:46513 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0450ab8807f5:0, corePoolSize=3, maxPoolSize=3 2024-11-16T12:55:05,681 INFO [RS:2;0450ab8807f5:46513 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:05,681 INFO [RS:2;0450ab8807f5:46513 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:05,681 INFO [RS:2;0450ab8807f5:46513 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:05,681 INFO [RS:2;0450ab8807f5:46513 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:05,681 INFO [RS:2;0450ab8807f5:46513 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:05,681 INFO [RS:2;0450ab8807f5:46513 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,46513,1731761705576-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T12:55:05,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T12:55:05,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731761703376, region=df427f7c2a516b6a4dc66ac07c6d145d, ASSIGN in 333 msec 2024-11-16T12:55:05,682 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 0450ab8807f5,41647,1731761702198 after splitting done 2024-11-16T12:55:05,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 0450ab8807f5,41647,1731761702198, splitWal=true, meta=false in 346 msec 2024-11-16T12:55:05,700 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T12:55:05,700 INFO [RS:2;0450ab8807f5:46513 {}] hbase.ChoreService(168): Chore ScheduledChore name=0450ab8807f5,46513,1731761705576-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:05,700 INFO [RS:2;0450ab8807f5:46513 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:05,701 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.Replication(171): 0450ab8807f5,46513,1731761705576 started 2024-11-16T12:55:05,714 INFO [RS:2;0450ab8807f5:46513 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T12:55:05,714 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.HRegionServer(1482): Serving as 0450ab8807f5,46513,1731761705576, RpcServer on 0450ab8807f5/172.17.0.2:46513, sessionid=0x1014500175b0003 2024-11-16T12:55:05,714 DEBUG [RS:2;0450ab8807f5:46513 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T12:55:05,714 DEBUG [RS:2;0450ab8807f5:46513 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0450ab8807f5,46513,1731761705576 2024-11-16T12:55:05,715 DEBUG [RS:2;0450ab8807f5:46513 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,46513,1731761705576' 2024-11-16T12:55:05,715 DEBUG [RS:2;0450ab8807f5:46513 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T12:55:05,715 DEBUG [RS:2;0450ab8807f5:46513 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T12:55:05,716 DEBUG [RS:2;0450ab8807f5:46513 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T12:55:05,716 DEBUG [RS:2;0450ab8807f5:46513 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T12:55:05,716 DEBUG [RS:2;0450ab8807f5:46513 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0450ab8807f5,46513,1731761705576 2024-11-16T12:55:05,716 DEBUG [RS:2;0450ab8807f5:46513 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0450ab8807f5,46513,1731761705576' 2024-11-16T12:55:05,716 DEBUG [RS:2;0450ab8807f5:46513 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T12:55:05,716 DEBUG [RS:2;0450ab8807f5:46513 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T12:55:05,717 DEBUG [RS:2;0450ab8807f5:46513 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T12:55:05,717 INFO [RS:2;0450ab8807f5:46513 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T12:55:05,717 INFO [RS:2;0450ab8807f5:46513 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T12:55:05,818 INFO [RS:2;0450ab8807f5:46513 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T12:55:05,820 INFO [RS:2;0450ab8807f5:46513 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0450ab8807f5%2C46513%2C1731761705576, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,46513,1731761705576, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/oldWALs, maxLogs=55 2024-11-16T12:55:05,845 INFO [RS:2;0450ab8807f5:46513 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/WALs/0450ab8807f5,46513,1731761705576/0450ab8807f5%2C46513%2C1731761705576.1731761705822 2024-11-16T12:55:05,846 DEBUG [RS:2;0450ab8807f5:46513 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T12:55:06,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T12:55:06,215 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T12:55:06,215 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:55:06,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:06,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:06,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T12:55:06,216 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T12:55:06,216 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1207918263, stopped=false 2024-11-16T12:55:06,216 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0450ab8807f5,38799,1731761702052 2024-11-16T12:55:06,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T12:55:06,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46513-0x1014500175b0003, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T12:55:06,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T12:55:06,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:06,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46513-0x1014500175b0003, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:06,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:06,256 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T12:55:06,257 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T12:55:06,257 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:55:06,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:06,258 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:55:06,258 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,36895,1731761702235' ***** 2024-11-16T12:55:06,258 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:55:06,258 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T12:55:06,258 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0450ab8807f5,46513,1731761705576' ***** 2024-11-16T12:55:06,258 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T12:55:06,258 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46513-0x1014500175b0003, quorum=127.0.0.1:53805, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T12:55:06,258 INFO [RS:1;0450ab8807f5:36895 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T12:55:06,259 INFO [RS:1;0450ab8807f5:36895 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/bucket.persistence 2024-11-16T12:55:06,259 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T12:55:06,259 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T12:55:06,259 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T12:55:06,259 INFO [RS:1;0450ab8807f5:36895 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T12:55:06,259 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T12:55:06,261 INFO [RS:2;0450ab8807f5:46513 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T12:55:06,261 INFO [RS:2;0450ab8807f5:46513 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02281b18-0223-1ff0-eb46-757a1231dd91/bucket.persistence 2024-11-16T12:55:06,261 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T12:55:06,261 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T12:55:06,261 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T12:55:06,261 INFO [RS:2;0450ab8807f5:46513 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T12:55:06,261 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T12:55:06,269 DEBUG [RS:1;0450ab8807f5:36895 {}] bucket.BucketCache(1487): Thread RS:1;0450ab8807f5:36895 started persisting bucket cache to file 2024-11-16T12:55:06,269 DEBUG [RS:1;0450ab8807f5:36895 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T12:55:06,269 DEBUG [RS:1;0450ab8807f5:36895 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T12:55:06,271 DEBUG [RS:2;0450ab8807f5:46513 {}] bucket.BucketCache(1487): Thread RS:2;0450ab8807f5:46513 started persisting bucket cache to file 2024-11-16T12:55:06,271 DEBUG [RS:2;0450ab8807f5:46513 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T12:55:06,272 DEBUG [RS:2;0450ab8807f5:46513 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-16T12:55:06,272 DEBUG [RS:1;0450ab8807f5:36895 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T12:55:06,272 DEBUG [RS:1;0450ab8807f5:36895 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731761706269 2024-11-16T12:55:06,272 DEBUG [RS:1;0450ab8807f5:36895 {}] bucket.BucketCache(1509): Thread RS:1;0450ab8807f5:36895 finished persisting bucket cache to file, renaming 2024-11-16T12:55:06,272 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:55:06,272 INFO [RS:1;0450ab8807f5:36895 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T12:55:06,272 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:55:06,272 INFO [RS:1;0450ab8807f5:36895 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T12:55:06,272 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer(3091): Received CLOSE for df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:06,272 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer(959): stopping server 0450ab8807f5,36895,1731761702235 2024-11-16T12:55:06,272 INFO [RS:1;0450ab8807f5:36895 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:55:06,273 INFO [RS:1;0450ab8807f5:36895 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0450ab8807f5:36895. 2024-11-16T12:55:06,273 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T12:55:06,273 DEBUG [RS:1;0450ab8807f5:36895 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:55:06,273 DEBUG [RS:1;0450ab8807f5:36895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:06,273 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing df427f7c2a516b6a4dc66ac07c6d145d, disabling compactions & flushes 2024-11-16T12:55:06,273 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:06,273 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T12:55:06,273 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:06,273 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T12:55:06,273 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. after waiting 0 ms 2024-11-16T12:55:06,273 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T12:55:06,273 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:06,273 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T12:55:06,273 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-16T12:55:06,273 DEBUG [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, df427f7c2a516b6a4dc66ac07c6d145d=testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d.} 2024-11-16T12:55:06,273 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T12:55:06,273 DEBUG [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, df427f7c2a516b6a4dc66ac07c6d145d 2024-11-16T12:55:06,273 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T12:55:06,274 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T12:55:06,274 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T12:55:06,274 DEBUG [StoreFileCloser-df427f7c2a516b6a4dc66ac07c6d145d-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/family/496bb1911f23400a8b95077d4757d038 2024-11-16T12:55:06,274 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T12:55:06,274 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T12:55:06,274 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-16T12:55:06,274 DEBUG [RS:2;0450ab8807f5:46513 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-16T12:55:06,274 DEBUG [RS:2;0450ab8807f5:46513 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731761706271 2024-11-16T12:55:06,274 DEBUG [RS:2;0450ab8807f5:46513 {}] bucket.BucketCache(1509): Thread RS:2;0450ab8807f5:46513 finished persisting bucket cache to file, renaming 2024-11-16T12:55:06,275 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T12:55:06,275 INFO [RS:2;0450ab8807f5:46513 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T12:55:06,275 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T12:55:06,275 INFO [RS:2;0450ab8807f5:46513 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T12:55:06,275 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.HRegionServer(959): stopping server 0450ab8807f5,46513,1731761705576 2024-11-16T12:55:06,275 INFO [RS:2;0450ab8807f5:46513 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:55:06,275 INFO [RS:2;0450ab8807f5:46513 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;0450ab8807f5:46513. 2024-11-16T12:55:06,275 DEBUG [RS:2;0450ab8807f5:46513 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T12:55:06,276 DEBUG [RS:2;0450ab8807f5:46513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:06,276 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.HRegionServer(976): stopping server 0450ab8807f5,46513,1731761705576; all regions closed. 2024-11-16T12:55:06,278 DEBUG [RS:2;0450ab8807f5:46513 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/oldWALs 2024-11-16T12:55:06,278 INFO [RS:2;0450ab8807f5:46513 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0450ab8807f5%2C46513%2C1731761705576:(num 1731761705822) 2024-11-16T12:55:06,278 DEBUG [RS:2;0450ab8807f5:46513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:06,278 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:55:06,278 INFO [RS:2;0450ab8807f5:46513 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:55:06,278 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:55:06,278 INFO [RS:2;0450ab8807f5:46513 {}] hbase.ChoreService(370): Chore service for: regionserver/0450ab8807f5:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T12:55:06,278 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T12:55:06,278 INFO [regionserver/0450ab8807f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:55:06,279 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T12:55:06,279 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/default/testBlockEvictionOnGracefulStop1731761703376/df427f7c2a516b6a4dc66ac07c6d145d/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-16T12:55:06,279 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T12:55:06,279 INFO [RS:2;0450ab8807f5:46513 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:55:06,279 INFO [RS:2;0450ab8807f5:46513 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46513 2024-11-16T12:55:06,279 INFO [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:06,279 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for df427f7c2a516b6a4dc66ac07c6d145d: Waiting for close lock at 1731761706273Running coprocessor pre-close hooks at 1731761706273Disabling compacts and flushes for region at 1731761706273Disabling writes for close at 1731761706273Writing region close event to WAL at 1731761706274 (+1 ms)Running coprocessor post-close hooks at 1731761706279 (+5 ms)Closed at 1731761706279 2024-11-16T12:55:06,279 DEBUG [RS_CLOSE_REGION-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d. 2024-11-16T12:55:06,282 INFO [regionserver/0450ab8807f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:55:06,285 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 89342043ebf2486b875c4665c9227648 2024-11-16T12:55:06,297 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/info/89342043ebf2486b875c4665c9227648 is 209, key is testBlockEvictionOnGracefulStop1731761703376,,1731761703382.df427f7c2a516b6a4dc66ac07c6d145d./info:regioninfo/1731761705674/Put/seqid=0 2024-11-16T12:55:06,297 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/info/89342043ebf2486b875c4665c9227648 2024-11-16T12:55:06,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46513-0x1014500175b0003, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0450ab8807f5,46513,1731761705576 2024-11-16T12:55:06,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:55:06,302 INFO [RS:2;0450ab8807f5:46513 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:55:06,302 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0450ab8807f5,46513,1731761705576] 2024-11-16T12:55:06,305 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 71179d143fd346b68fde4be44423765c 2024-11-16T12:55:06,316 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/ns/71179d143fd346b68fde4be44423765c is 43, key is default/ns:d/1731761703304/Put/seqid=0 2024-11-16T12:55:06,316 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/ns/71179d143fd346b68fde4be44423765c 2024-11-16T12:55:06,318 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0450ab8807f5,46513,1731761705576 already deleted, retry=false 2024-11-16T12:55:06,318 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0450ab8807f5,46513,1731761705576 expired; onlineServers=1 2024-11-16T12:55:06,323 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 11a1ca4443834a9ea1fda689c96c20ed 2024-11-16T12:55:06,336 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/table/11a1ca4443834a9ea1fda689c96c20ed is 80, key is testBlockEvictionOnGracefulStop1731761703376/table:state/1731761703764/Put/seqid=0 2024-11-16T12:55:06,337 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/table/11a1ca4443834a9ea1fda689c96c20ed 2024-11-16T12:55:06,339 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/info/89342043ebf2486b875c4665c9227648 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/info/89342043ebf2486b875c4665c9227648 2024-11-16T12:55:06,341 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/info/89342043ebf2486b875c4665c9227648, delay=950 ms 2024-11-16T12:55:06,342 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/info/89342043ebf2486b875c4665c9227648, entries=14, sequenceid=13, filesize=7.5 K 2024-11-16T12:55:06,342 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/ns/71179d143fd346b68fde4be44423765c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/ns/71179d143fd346b68fde4be44423765c 2024-11-16T12:55:06,343 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/ns/71179d143fd346b68fde4be44423765c, delay=993 ms 2024-11-16T12:55:06,345 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/ns/71179d143fd346b68fde4be44423765c, entries=2, sequenceid=13, filesize=5.0 K 2024-11-16T12:55:06,345 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/.tmp/table/11a1ca4443834a9ea1fda689c96c20ed as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/table/11a1ca4443834a9ea1fda689c96c20ed 2024-11-16T12:55:06,347 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/table/11a1ca4443834a9ea1fda689c96c20ed, delay=991 ms 2024-11-16T12:55:06,348 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/table/11a1ca4443834a9ea1fda689c96c20ed, entries=2, sequenceid=13, filesize=5.3 K 2024-11-16T12:55:06,349 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=13, compaction requested=false 2024-11-16T12:55:06,349 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/info/89342043ebf2486b875c4665c9227648 2024-11-16T12:55:06,350 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/ns/71179d143fd346b68fde4be44423765c 2024-11-16T12:55:06,350 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/table/11a1ca4443834a9ea1fda689c96c20ed 2024-11-16T12:55:06,351 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T12:55:06,352 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T12:55:06,353 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T12:55:06,353 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T12:55:06,353 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T12:55:06,353 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T12:55:06,354 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-16T12:55:06,354 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T12:55:06,355 INFO [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T12:55:06,355 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731761706273Running coprocessor pre-close hooks at 1731761706273Disabling compacts and flushes for region at 1731761706273Disabling writes for close at 1731761706274 (+1 ms)Obtaining lock to block concurrent updates at 1731761706274Preparing flush snapshotting stores in 1588230740 at 1731761706274Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1731761706274Flushing stores of hbase:meta,,1.1588230740 at 1731761706275 (+1 ms)Flushing 1588230740/info: creating writer at 1731761706275Flushing 1588230740/info: appending metadata at 1731761706296 (+21 ms)Flushing 1588230740/info: closing flushed file at 1731761706296Flushing 1588230740/ns: creating writer at 1731761706299 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731761706315 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731761706315Flushing 1588230740/table: creating writer at 1731761706318 (+3 ms)Flushing 1588230740/table: appending metadata at 1731761706335 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731761706335Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23378ece: reopening flushed file at 1731761706339 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fb3723a: reopening flushed file at 1731761706342 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11b6b2dc: reopening flushed file at 1731761706345 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=13, compaction requested=false at 1731761706349 (+4 ms)Writing region close event to WAL at 1731761706351 (+2 ms)Running coprocessor post-close hooks at 1731761706354 (+3 ms)Closed at 1731761706355 (+1 ms) 2024-11-16T12:55:06,355 DEBUG [RS_CLOSE_META-regionserver/0450ab8807f5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T12:55:06,410 INFO [RS:2;0450ab8807f5:46513 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:55:06,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46513-0x1014500175b0003, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:55:06,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46513-0x1014500175b0003, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:55:06,410 INFO [RS:2;0450ab8807f5:46513 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,46513,1731761705576; zookeeper connection closed. 2024-11-16T12:55:06,411 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7d03258c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7d03258c 2024-11-16T12:55:06,474 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer(976): stopping server 0450ab8807f5,36895,1731761702235; all regions closed. 2024-11-16T12:55:06,478 DEBUG [RS:1;0450ab8807f5:36895 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/oldWALs 2024-11-16T12:55:06,478 INFO [RS:1;0450ab8807f5:36895 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0450ab8807f5%2C36895%2C1731761702235.meta:.meta(num 1731761703258) 2024-11-16T12:55:06,481 DEBUG [RS:1;0450ab8807f5:36895 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/oldWALs 2024-11-16T12:55:06,481 INFO [RS:1;0450ab8807f5:36895 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0450ab8807f5%2C36895%2C1731761702235:(num 1731761702907) 2024-11-16T12:55:06,481 DEBUG [RS:1;0450ab8807f5:36895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T12:55:06,481 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T12:55:06,482 INFO [RS:1;0450ab8807f5:36895 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:55:06,482 INFO [RS:1;0450ab8807f5:36895 {}] hbase.ChoreService(370): Chore service for: regionserver/0450ab8807f5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T12:55:06,483 INFO [RS:1;0450ab8807f5:36895 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:55:06,483 INFO [regionserver/0450ab8807f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:55:06,483 INFO [RS:1;0450ab8807f5:36895 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36895 2024-11-16T12:55:06,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0450ab8807f5,36895,1731761702235 2024-11-16T12:55:06,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T12:55:06,510 INFO [RS:1;0450ab8807f5:36895 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:55:06,511 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0450ab8807f5,36895,1731761702235] 2024-11-16T12:55:06,527 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0450ab8807f5,36895,1731761702235 already deleted, retry=false 2024-11-16T12:55:06,527 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0450ab8807f5,36895,1731761702235 expired; onlineServers=0 2024-11-16T12:55:06,527 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0450ab8807f5,38799,1731761702052' ***** 2024-11-16T12:55:06,527 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T12:55:06,527 INFO [M:0;0450ab8807f5:38799 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T12:55:06,527 INFO [M:0;0450ab8807f5:38799 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T12:55:06,528 DEBUG [M:0;0450ab8807f5:38799 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T12:55:06,528 DEBUG [M:0;0450ab8807f5:38799 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T12:55:06,528 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T12:55:06,528 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761702612 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.small.0-1731761702612,5,FailOnTimeoutGroup] 2024-11-16T12:55:06,528 DEBUG [master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761702612 {}] cleaner.HFileCleaner(306): Exit Thread[master/0450ab8807f5:0:becomeActiveMaster-HFileCleaner.large.0-1731761702612,5,FailOnTimeoutGroup] 2024-11-16T12:55:06,528 INFO [M:0;0450ab8807f5:38799 {}] hbase.ChoreService(370): Chore service for: master/0450ab8807f5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T12:55:06,529 INFO [M:0;0450ab8807f5:38799 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T12:55:06,529 DEBUG [M:0;0450ab8807f5:38799 {}] master.HMaster(1795): Stopping service threads 2024-11-16T12:55:06,529 INFO [M:0;0450ab8807f5:38799 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T12:55:06,529 INFO [M:0;0450ab8807f5:38799 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T12:55:06,530 INFO [M:0;0450ab8807f5:38799 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T12:55:06,530 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T12:55:06,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T12:55:06,535 DEBUG [M:0;0450ab8807f5:38799 {}] zookeeper.ZKUtil(347): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T12:55:06,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T12:55:06,535 WARN [M:0;0450ab8807f5:38799 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T12:55:06,535 INFO [M:0;0450ab8807f5:38799 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/.lastflushedseqids 2024-11-16T12:55:06,542 INFO [M:0;0450ab8807f5:38799 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T12:55:06,542 INFO [M:0;0450ab8807f5:38799 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T12:55:06,542 DEBUG [M:0;0450ab8807f5:38799 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T12:55:06,542 INFO [M:0;0450ab8807f5:38799 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:55:06,542 DEBUG [M:0;0450ab8807f5:38799 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:55:06,542 DEBUG [M:0;0450ab8807f5:38799 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T12:55:06,542 DEBUG [M:0;0450ab8807f5:38799 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:55:06,542 INFO [M:0;0450ab8807f5:38799 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.84 KB heapSize=41.09 KB 2024-11-16T12:55:06,552 DEBUG [M:0;0450ab8807f5:38799 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fc843b6f8fe44bc3ba6a3c43d762ad82 2024-11-16T12:55:06,567 DEBUG [M:0;0450ab8807f5:38799 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fc843b6f8fe44bc3ba6a3c43d762ad82 is 82, key is hbase:meta,,1/info:regioninfo/1731761703287/Put/seqid=0 2024-11-16T12:55:06,567 INFO [M:0;0450ab8807f5:38799 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fc843b6f8fe44bc3ba6a3c43d762ad82 2024-11-16T12:55:06,574 DEBUG [M:0;0450ab8807f5:38799 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3748bc3437f84029a1487e864e984fe9 2024-11-16T12:55:06,586 DEBUG [M:0;0450ab8807f5:38799 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3748bc3437f84029a1487e864e984fe9 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731761703768/Put/seqid=0 2024-11-16T12:55:06,587 INFO [M:0;0450ab8807f5:38799 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.10 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3748bc3437f84029a1487e864e984fe9 2024-11-16T12:55:06,593 DEBUG [M:0;0450ab8807f5:38799 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7a5dfdbc22ed4cb9bd1b8d79d3fb30b3 2024-11-16T12:55:06,605 DEBUG [M:0;0450ab8807f5:38799 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7a5dfdbc22ed4cb9bd1b8d79d3fb30b3 is 69, key is 0450ab8807f5,36895,1731761702235/rs:state/1731761702715/Put/seqid=0 2024-11-16T12:55:06,605 INFO [M:0;0450ab8807f5:38799 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7a5dfdbc22ed4cb9bd1b8d79d3fb30b3 2024-11-16T12:55:06,607 INFO [M:0;0450ab8807f5:38799 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7a5dfdbc22ed4cb9bd1b8d79d3fb30b3 2024-11-16T12:55:06,607 DEBUG [M:0;0450ab8807f5:38799 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fc843b6f8fe44bc3ba6a3c43d762ad82 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fc843b6f8fe44bc3ba6a3c43d762ad82 2024-11-16T12:55:06,609 INFO [M:0;0450ab8807f5:38799 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fc843b6f8fe44bc3ba6a3c43d762ad82, entries=8, sequenceid=83, filesize=5.5 K 2024-11-16T12:55:06,609 DEBUG [M:0;0450ab8807f5:38799 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3748bc3437f84029a1487e864e984fe9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3748bc3437f84029a1487e864e984fe9 2024-11-16T12:55:06,610 INFO [M:0;0450ab8807f5:38799 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3748bc3437f84029a1487e864e984fe9, entries=9, sequenceid=83, filesize=6.4 K 2024-11-16T12:55:06,611 DEBUG [M:0;0450ab8807f5:38799 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7a5dfdbc22ed4cb9bd1b8d79d3fb30b3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7a5dfdbc22ed4cb9bd1b8d79d3fb30b3 2024-11-16T12:55:06,612 INFO [M:0;0450ab8807f5:38799 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7a5dfdbc22ed4cb9bd1b8d79d3fb30b3 2024-11-16T12:55:06,612 INFO [M:0;0450ab8807f5:38799 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d781a88a-dcc5-b0d4-ac62-fa6ce09d907b/b5f90c5c-7786-5aee-ba7e-846c57b09b3a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7a5dfdbc22ed4cb9bd1b8d79d3fb30b3, entries=3, sequenceid=83, filesize=5.3 K 2024-11-16T12:55:06,613 INFO [M:0;0450ab8807f5:38799 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=83, compaction requested=false 2024-11-16T12:55:06,615 INFO [M:0;0450ab8807f5:38799 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T12:55:06,615 DEBUG [M:0;0450ab8807f5:38799 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731761706542Disabling compacts and flushes for region at 1731761706542Disabling writes for close at 1731761706542Obtaining lock to block concurrent updates at 1731761706542Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731761706542Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33626, getHeapSize=42008, getOffHeapSize=0, getCellsCount=98 at 1731761706543 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731761706543Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731761706543Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731761706566 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731761706566Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731761706569 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731761706586 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731761706586Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731761706588 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731761706604 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731761706604Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63550249: reopening flushed file at 1731761706607 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@390c7042: reopening flushed file at 1731761706609 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4aa36919: reopening flushed file at 1731761706611 (+2 ms)Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=83, compaction requested=false at 1731761706613 (+2 ms)Writing region close event to WAL at 1731761706615 (+2 ms)Closed at 1731761706615 2024-11-16T12:55:06,616 INFO [M:0;0450ab8807f5:38799 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T12:55:06,616 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T12:55:06,616 INFO [M:0;0450ab8807f5:38799 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38799 2024-11-16T12:55:06,616 INFO [M:0;0450ab8807f5:38799 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T12:55:06,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:55:06,619 INFO [RS:1;0450ab8807f5:36895 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:55:06,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1014500175b0002, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:55:06,619 INFO [RS:1;0450ab8807f5:36895 {}] regionserver.HRegionServer(1031): Exiting; stopping=0450ab8807f5,36895,1731761702235; zookeeper connection closed. 2024-11-16T12:55:06,619 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3fa5e156 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3fa5e156 2024-11-16T12:55:06,619 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-16T12:55:06,742 INFO [M:0;0450ab8807f5:38799 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T12:55:06,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:55:06,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38799-0x1014500175b0000, quorum=127.0.0.1:53805, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T12:55:06,749 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T12:55:06,751 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-16T12:55:06,751 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T12:55:06,759 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 408), ProcessCount=11 (was 11), AvailableMemoryMB=3995 (was 4066)