2024-12-01 23:56:22,084 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-01 23:56:22,104 main DEBUG Took 0.011523 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-01 23:56:22,104 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-01 23:56:22,105 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-01 23:56:22,105 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-01 23:56:22,107 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:56:22,114 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-01 23:56:22,130 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:56:22,131 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:56:22,132 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:56:22,132 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:56:22,133 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:56:22,133 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:56:22,134 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:56:22,135 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:56:22,136 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:56:22,136 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:56:22,137 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:56:22,137 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:56:22,138 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:56:22,138 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:56:22,139 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:56:22,139 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:56:22,140 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:56:22,140 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:56:22,140 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:56:22,141 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:56:22,141 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:56:22,142 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:56:22,142 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:56:22,142 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:56:22,143 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:56:22,143 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-01 23:56:22,145 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:56:22,146 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-01 23:56:22,148 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-01 23:56:22,149 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-01 23:56:22,151 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-01 23:56:22,151 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-01 23:56:22,161 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-01 23:56:22,164 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-01 23:56:22,166 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-01 23:56:22,166 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-01 23:56:22,166 main DEBUG createAppenders(={Console}) 2024-12-01 23:56:22,167 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-01 23:56:22,167 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-01 23:56:22,168 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-01 23:56:22,168 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-01 23:56:22,169 main DEBUG OutputStream closed 2024-12-01 23:56:22,169 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-01 23:56:22,169 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-01 23:56:22,170 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-01 23:56:22,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-01 23:56:22,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-01 23:56:22,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-01 23:56:22,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-01 23:56:22,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-01 23:56:22,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-01 23:56:22,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-01 23:56:22,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-01 23:56:22,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-01 23:56:22,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-01 23:56:22,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-01 23:56:22,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-01 23:56:22,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-01 23:56:22,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-01 23:56:22,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-01 23:56:22,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-01 23:56:22,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-01 23:56:22,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-01 23:56:22,255 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-01 23:56:22,255 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-01 23:56:22,255 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-01 23:56:22,256 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-01T23:56:22,267 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-01 23:56:22,270 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-01 23:56:22,270 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-01T23:56:22,470 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d 2024-12-01T23:56:22,493 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=389, ProcessCount=11, AvailableMemoryMB=7899 2024-12-01T23:56:22,585 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-01T23:56:22,668 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:22,681 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/cluster_83cd5761-1ce7-ac29-d399-5dbdd05bc812, deleteOnExit=true 2024-12-01T23:56:22,802 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/cluster_83cd5761-1ce7-ac29-d399-5dbdd05bc812/zookeeper_0, clientPort=49794, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/cluster_83cd5761-1ce7-ac29-d399-5dbdd05bc812/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/cluster_83cd5761-1ce7-ac29-d399-5dbdd05bc812/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-01T23:56:22,812 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49794 2024-12-01T23:56:22,812 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:22,886 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357 with version=8 2024-12-01T23:56:22,887 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/hbase-staging 2024-12-01T23:56:22,968 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-01T23:56:23,212 INFO [Time-limited test {}] client.ConnectionUtils(128): master/579b0f681375:0 server-side Connection retries=45 2024-12-01T23:56:23,220 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:23,221 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:23,231 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T23:56:23,231 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:23,232 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T23:56:23,365 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-01T23:56:23,448 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T23:56:23,480 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-01T23:56:23,489 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-01T23:56:23,493 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T23:56:23,529 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 109649 (auto-detected) 2024-12-01T23:56:23,530 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-01T23:56:23,553 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35455 2024-12-01T23:56:23,578 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35455 connecting to ZooKeeper ensemble=127.0.0.1:49794 2024-12-01T23:56:23,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:354550x0, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T23:56:23,697 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35455-0x101949cf63e0000 connected 2024-12-01T23:56:23,785 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:23,786 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:23,796 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:56:23,800 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357, hbase.cluster.distributed=false 2024-12-01T23:56:23,821 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T23:56:23,826 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35455 2024-12-01T23:56:23,827 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35455 2024-12-01T23:56:23,827 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35455 2024-12-01T23:56:23,828 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35455 2024-12-01T23:56:23,828 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35455 2024-12-01T23:56:23,900 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/579b0f681375:0 server-side Connection retries=45 2024-12-01T23:56:23,901 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:23,901 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:23,902 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T23:56:23,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:23,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T23:56:23,905 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T23:56:23,907 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T23:56:23,908 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36617 2024-12-01T23:56:23,910 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36617 connecting to ZooKeeper ensemble=127.0.0.1:49794 2024-12-01T23:56:23,910 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:23,911 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:23,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:366170x0, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T23:56:23,930 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36617-0x101949cf63e0001 connected 2024-12-01T23:56:23,930 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:56:23,937 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T23:56:23,949 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/bucket.cache 2024-12-01T23:56:23,954 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-01T23:56:23,956 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-01T23:56:23,957 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-01T23:56:23,958 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-01T23:56:23,959 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-01T23:56:23,963 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-01T23:56:23,966 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-01T23:56:23,967 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/bucket.persistence is complete. 2024-12-01T23:56:23,968 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T23:56:23,971 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T23:56:23,976 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T23:56:23,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36617 2024-12-01T23:56:23,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36617 2024-12-01T23:56:23,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36617 2024-12-01T23:56:23,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36617 2024-12-01T23:56:23,980 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36617 2024-12-01T23:56:23,996 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/579b0f681375:0 server-side Connection retries=45 2024-12-01T23:56:23,996 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:23,997 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:23,997 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T23:56:23,997 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:23,997 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T23:56:23,997 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T23:56:23,998 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T23:56:23,999 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37553 2024-12-01T23:56:24,001 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37553 connecting to ZooKeeper ensemble=127.0.0.1:49794 2024-12-01T23:56:24,001 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:24,002 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:24,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:375530x0, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T23:56:24,031 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:375530x0, quorum=127.0.0.1:49794, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:56:24,031 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37553-0x101949cf63e0002 connected 2024-12-01T23:56:24,032 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T23:56:24,033 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/bucket.cache 2024-12-01T23:56:24,033 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-01T23:56:24,034 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-01T23:56:24,034 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-01T23:56:24,034 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-01T23:56:24,034 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-01T23:56:24,035 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-01T23:56:24,035 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-01T23:56:24,035 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/bucket.persistence is complete. 2024-12-01T23:56:24,035 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T23:56:24,037 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T23:56:24,040 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T23:56:24,042 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37553 2024-12-01T23:56:24,043 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37553 2024-12-01T23:56:24,045 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37553 2024-12-01T23:56:24,046 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37553 2024-12-01T23:56:24,047 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37553 2024-12-01T23:56:24,067 DEBUG [M:0;579b0f681375:35455 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;579b0f681375:35455 2024-12-01T23:56:24,068 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/579b0f681375,35455,1733097383050 2024-12-01T23:56:24,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:56:24,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:56:24,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:56:24,083 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/579b0f681375,35455,1733097383050 2024-12-01T23:56:24,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:24,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T23:56:24,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T23:56:24,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:24,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:24,114 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-01T23:56:24,115 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/579b0f681375,35455,1733097383050 from backup master directory 2024-12-01T23:56:24,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:56:24,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:56:24,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/579b0f681375,35455,1733097383050 2024-12-01T23:56:24,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:56:24,122 WARN [master/579b0f681375:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T23:56:24,122 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=579b0f681375,35455,1733097383050 2024-12-01T23:56:24,124 INFO [master/579b0f681375:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-01T23:56:24,126 INFO [master/579b0f681375:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-01T23:56:24,134 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/hbase.id] with ID: c7c053aa-d823-4a37-ab05-ab423c4adf45 2024-12-01T23:56:24,134 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/.tmp/hbase.id 2024-12-01T23:56:24,140 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/hbase.id] 2024-12-01T23:56:24,178 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:24,183 INFO [master/579b0f681375:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-01T23:56:24,200 INFO [master/579b0f681375:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-01T23:56:24,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:24,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:24,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:24,251 INFO [master/579b0f681375:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T23:56:24,254 INFO [master/579b0f681375:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-01T23:56:24,290 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:56:24,294 INFO [master/579b0f681375:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:56:24,342 INFO [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store 2024-12-01T23:56:24,361 INFO [master/579b0f681375:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-01T23:56:24,365 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:56:24,366 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T23:56:24,366 INFO [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:56:24,367 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:56:24,368 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T23:56:24,369 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:56:24,369 INFO [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:56:24,370 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733097384366Disabling compacts and flushes for region at 1733097384366Disabling writes for close at 1733097384369 (+3 ms)Writing region close event to WAL at 1733097384369Closed at 1733097384369 2024-12-01T23:56:24,370 WARN [master/579b0f681375:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/.initializing 2024-12-01T23:56:24,370 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/WALs/579b0f681375,35455,1733097383050 2024-12-01T23:56:24,382 INFO [master/579b0f681375:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:56:24,395 INFO [master/579b0f681375:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=579b0f681375%2C35455%2C1733097383050, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/WALs/579b0f681375,35455,1733097383050, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/oldWALs, maxLogs=10 2024-12-01T23:56:24,431 INFO [master/579b0f681375:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/WALs/579b0f681375,35455,1733097383050/579b0f681375%2C35455%2C1733097383050.1733097384399 2024-12-01T23:56:24,432 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T23:56:24,433 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:56:24,433 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:56:24,437 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:24,438 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:24,476 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:24,507 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-01T23:56:24,512 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:24,514 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:24,515 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:24,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-01T23:56:24,521 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:24,522 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:56:24,522 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:24,528 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-01T23:56:24,529 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:24,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:56:24,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:24,534 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-01T23:56:24,535 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:24,535 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:56:24,536 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:24,540 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:24,541 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:24,545 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:24,546 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:24,550 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T23:56:24,553 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:24,561 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:56:24,562 INFO [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71002339, jitterRate=0.058017298579216}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T23:56:24,571 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733097384450Initializing all the Stores at 1733097384451 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097384451Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733097384452 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733097384453 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733097384453Cleaning up temporary data from old regions at 1733097384547 (+94 ms)Region opened successfully at 1733097384571 (+24 ms) 2024-12-01T23:56:24,573 INFO [master/579b0f681375:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-01T23:56:24,608 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dbbf432, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=579b0f681375/172.17.0.2:0 2024-12-01T23:56:24,644 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-01T23:56:24,657 INFO [master/579b0f681375:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-01T23:56:24,657 INFO [master/579b0f681375:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-01T23:56:24,661 INFO [master/579b0f681375:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-01T23:56:24,661 INFO [master/579b0f681375:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-01T23:56:24,667 INFO [master/579b0f681375:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-01T23:56:24,668 INFO [master/579b0f681375:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-01T23:56:24,695 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-01T23:56:24,707 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-01T23:56:24,729 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-01T23:56:24,732 INFO [master/579b0f681375:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-01T23:56:24,734 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-01T23:56:24,737 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-01T23:56:24,739 INFO [master/579b0f681375:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-01T23:56:24,743 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-01T23:56:24,754 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-01T23:56:24,756 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-01T23:56:24,762 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-01T23:56:24,785 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-01T23:56:24,796 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-01T23:56:24,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T23:56:24,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T23:56:24,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T23:56:24,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:24,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:24,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:24,816 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=579b0f681375,35455,1733097383050, sessionid=0x101949cf63e0000, setting cluster-up flag (Was=false) 2024-12-01T23:56:24,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:24,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:24,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:24,863 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-01T23:56:24,866 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=579b0f681375,35455,1733097383050 2024-12-01T23:56:24,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:24,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:24,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:24,912 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-01T23:56:24,915 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=579b0f681375,35455,1733097383050 2024-12-01T23:56:24,919 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-01T23:56:24,953 INFO [RS:1;579b0f681375:37553 {}] regionserver.HRegionServer(746): ClusterId : c7c053aa-d823-4a37-ab05-ab423c4adf45 2024-12-01T23:56:24,953 INFO [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer(746): ClusterId : c7c053aa-d823-4a37-ab05-ab423c4adf45 2024-12-01T23:56:24,956 DEBUG [RS:1;579b0f681375:37553 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T23:56:24,956 DEBUG [RS:0;579b0f681375:36617 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T23:56:24,965 DEBUG [RS:0;579b0f681375:36617 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T23:56:24,965 DEBUG [RS:0;579b0f681375:36617 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T23:56:24,966 DEBUG [RS:1;579b0f681375:37553 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T23:56:24,966 DEBUG [RS:1;579b0f681375:37553 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T23:56:24,980 DEBUG [RS:0;579b0f681375:36617 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T23:56:24,980 DEBUG [RS:1;579b0f681375:37553 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T23:56:24,981 DEBUG [RS:1;579b0f681375:37553 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b87d2aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=579b0f681375/172.17.0.2:0 2024-12-01T23:56:24,981 DEBUG [RS:0;579b0f681375:36617 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60cfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=579b0f681375/172.17.0.2:0 2024-12-01T23:56:24,994 DEBUG [RS:1;579b0f681375:37553 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;579b0f681375:37553 2024-12-01T23:56:24,998 DEBUG [RS:0;579b0f681375:36617 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;579b0f681375:36617 2024-12-01T23:56:24,998 INFO [RS:1;579b0f681375:37553 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T23:56:24,998 INFO [RS:0;579b0f681375:36617 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T23:56:24,998 INFO [RS:1;579b0f681375:37553 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T23:56:24,998 INFO [RS:0;579b0f681375:36617 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T23:56:24,998 DEBUG [RS:1;579b0f681375:37553 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T23:56:24,998 DEBUG [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T23:56:25,002 INFO [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer(2659): reportForDuty to master=579b0f681375,35455,1733097383050 with port=36617, startcode=1733097383869 2024-12-01T23:56:25,002 INFO [RS:1;579b0f681375:37553 {}] regionserver.HRegionServer(2659): reportForDuty to master=579b0f681375,35455,1733097383050 with port=37553, startcode=1733097383995 2024-12-01T23:56:25,005 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-01T23:56:25,013 DEBUG [RS:0;579b0f681375:36617 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T23:56:25,013 DEBUG [RS:1;579b0f681375:37553 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T23:56:25,016 INFO [master/579b0f681375:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-01T23:56:25,024 INFO [master/579b0f681375:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-01T23:56:25,030 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 579b0f681375,35455,1733097383050 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-01T23:56:25,038 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/579b0f681375:0, corePoolSize=5, maxPoolSize=5 2024-12-01T23:56:25,038 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/579b0f681375:0, corePoolSize=5, maxPoolSize=5 2024-12-01T23:56:25,038 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/579b0f681375:0, corePoolSize=5, maxPoolSize=5 2024-12-01T23:56:25,038 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/579b0f681375:0, corePoolSize=5, maxPoolSize=5 2024-12-01T23:56:25,039 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/579b0f681375:0, corePoolSize=10, maxPoolSize=10 2024-12-01T23:56:25,039 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,039 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/579b0f681375:0, corePoolSize=2, maxPoolSize=2 2024-12-01T23:56:25,040 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,047 INFO [master/579b0f681375:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733097415047 2024-12-01T23:56:25,050 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-01T23:56:25,051 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-01T23:56:25,052 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T23:56:25,053 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-01T23:56:25,055 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-01T23:56:25,056 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-01T23:56:25,056 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-01T23:56:25,056 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-01T23:56:25,060 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:25,060 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-01T23:56:25,064 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,071 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-01T23:56:25,072 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-01T23:56:25,073 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-01T23:56:25,079 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-01T23:56:25,080 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-01T23:56:25,084 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/579b0f681375:0:becomeActiveMaster-HFileCleaner.large.0-1733097385081,5,FailOnTimeoutGroup] 2024-12-01T23:56:25,086 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/579b0f681375:0:becomeActiveMaster-HFileCleaner.small.0-1733097385085,5,FailOnTimeoutGroup] 2024-12-01T23:56:25,086 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,086 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-01T23:56:25,087 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,088 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,090 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45451, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T23:56:25,090 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45685, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T23:56:25,096 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35455 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 579b0f681375,36617,1733097383869 2024-12-01T23:56:25,098 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35455 {}] master.ServerManager(517): Registering regionserver=579b0f681375,36617,1733097383869 2024-12-01T23:56:25,102 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-01T23:56:25,102 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357 2024-12-01T23:56:25,109 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35455 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 579b0f681375,37553,1733097383995 2024-12-01T23:56:25,110 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35455 {}] master.ServerManager(517): Registering regionserver=579b0f681375,37553,1733097383995 2024-12-01T23:56:25,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:56:25,113 DEBUG [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357 2024-12-01T23:56:25,113 DEBUG [RS:1;579b0f681375:37553 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357 2024-12-01T23:56:25,113 DEBUG [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-01T23:56:25,114 DEBUG [RS:1;579b0f681375:37553 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-01T23:56:25,114 DEBUG [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T23:56:25,114 DEBUG [RS:1;579b0f681375:37553 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T23:56:25,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-01T23:56:25,123 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T23:56:25,124 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:25,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:25,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-01T23:56:25,129 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T23:56:25,129 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:25,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:25,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-01T23:56:25,134 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T23:56:25,134 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:25,134 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:25,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-01T23:56:25,139 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T23:56:25,140 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:25,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:25,140 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T23:56:25,141 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740 2024-12-01T23:56:25,142 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740 2024-12-01T23:56:25,142 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T23:56:25,142 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T23:56:25,143 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T23:56:25,143 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T23:56:25,150 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:56:25,151 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64837538, jitterRate=-0.03384539484977722}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T23:56:25,153 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733097385113Initializing all the Stores at 1733097385114 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097385114Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097385114Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733097385114Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097385115 (+1 ms)Cleaning up temporary data from old regions at 1733097385142 (+27 ms)Region opened successfully at 1733097385153 (+11 ms) 2024-12-01T23:56:25,153 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T23:56:25,154 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T23:56:25,154 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T23:56:25,154 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T23:56:25,154 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T23:56:25,155 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T23:56:25,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733097385153Disabling compacts and flushes for region at 1733097385153Disabling writes for close at 1733097385154 (+1 ms)Writing region close event to WAL at 1733097385155 (+1 ms)Closed at 1733097385155 2024-12-01T23:56:25,157 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T23:56:25,157 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-01T23:56:25,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-01T23:56:25,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T23:56:25,171 DEBUG [RS:1;579b0f681375:37553 {}] zookeeper.ZKUtil(111): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/579b0f681375,37553,1733097383995 2024-12-01T23:56:25,171 DEBUG [RS:0;579b0f681375:36617 {}] zookeeper.ZKUtil(111): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/579b0f681375,36617,1733097383869 2024-12-01T23:56:25,171 WARN [RS:1;579b0f681375:37553 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T23:56:25,171 WARN [RS:0;579b0f681375:36617 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T23:56:25,171 INFO [RS:1;579b0f681375:37553 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:56:25,171 INFO [RS:0;579b0f681375:36617 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:56:25,172 DEBUG [RS:1;579b0f681375:37553 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,37553,1733097383995 2024-12-01T23:56:25,172 DEBUG [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,36617,1733097383869 2024-12-01T23:56:25,174 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T23:56:25,176 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [579b0f681375,36617,1733097383869] 2024-12-01T23:56:25,176 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [579b0f681375,37553,1733097383995] 2024-12-01T23:56:25,177 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-01T23:56:25,199 INFO [RS:1;579b0f681375:37553 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T23:56:25,199 INFO [RS:0;579b0f681375:36617 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T23:56:25,216 INFO [RS:0;579b0f681375:36617 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T23:56:25,216 INFO [RS:1;579b0f681375:37553 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T23:56:25,221 INFO [RS:0;579b0f681375:36617 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T23:56:25,221 INFO [RS:1;579b0f681375:37553 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T23:56:25,221 INFO [RS:0;579b0f681375:36617 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,221 INFO [RS:1;579b0f681375:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,223 INFO [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T23:56:25,223 INFO [RS:1;579b0f681375:37553 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T23:56:25,228 INFO [RS:0;579b0f681375:36617 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T23:56:25,228 INFO [RS:1;579b0f681375:37553 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T23:56:25,230 INFO [RS:0;579b0f681375:36617 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,230 INFO [RS:1;579b0f681375:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,230 DEBUG [RS:0;579b0f681375:36617 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,230 DEBUG [RS:1;579b0f681375:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,231 DEBUG [RS:0;579b0f681375:36617 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,231 DEBUG [RS:1;579b0f681375:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,231 DEBUG [RS:0;579b0f681375:36617 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,231 DEBUG [RS:1;579b0f681375:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,231 DEBUG [RS:0;579b0f681375:36617 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,231 DEBUG [RS:1;579b0f681375:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,231 DEBUG [RS:0;579b0f681375:36617 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,231 DEBUG [RS:0;579b0f681375:36617 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/579b0f681375:0, corePoolSize=2, maxPoolSize=2 2024-12-01T23:56:25,231 DEBUG [RS:1;579b0f681375:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,231 DEBUG [RS:0;579b0f681375:36617 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,231 DEBUG [RS:1;579b0f681375:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/579b0f681375:0, corePoolSize=2, maxPoolSize=2 2024-12-01T23:56:25,231 DEBUG [RS:0;579b0f681375:36617 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,231 DEBUG [RS:0;579b0f681375:36617 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,231 DEBUG [RS:1;579b0f681375:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,232 DEBUG [RS:0;579b0f681375:36617 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,232 DEBUG [RS:1;579b0f681375:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,232 DEBUG [RS:0;579b0f681375:36617 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,232 DEBUG [RS:0;579b0f681375:36617 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,232 DEBUG [RS:1;579b0f681375:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,232 DEBUG [RS:0;579b0f681375:36617 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/579b0f681375:0, corePoolSize=3, maxPoolSize=3 2024-12-01T23:56:25,232 DEBUG [RS:1;579b0f681375:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,232 DEBUG [RS:0;579b0f681375:36617 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/579b0f681375:0, corePoolSize=3, maxPoolSize=3 2024-12-01T23:56:25,232 DEBUG [RS:1;579b0f681375:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,232 DEBUG [RS:1;579b0f681375:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:25,233 DEBUG [RS:1;579b0f681375:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/579b0f681375:0, corePoolSize=3, maxPoolSize=3 2024-12-01T23:56:25,233 DEBUG [RS:1;579b0f681375:37553 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/579b0f681375:0, corePoolSize=3, maxPoolSize=3 2024-12-01T23:56:25,237 INFO [RS:1;579b0f681375:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,237 INFO [RS:0;579b0f681375:36617 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,237 INFO [RS:1;579b0f681375:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,237 INFO [RS:0;579b0f681375:36617 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,237 INFO [RS:0;579b0f681375:36617 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,237 INFO [RS:0;579b0f681375:36617 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,237 INFO [RS:0;579b0f681375:36617 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,237 INFO [RS:0;579b0f681375:36617 {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,36617,1733097383869-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T23:56:25,238 INFO [RS:1;579b0f681375:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,238 INFO [RS:1;579b0f681375:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,238 INFO [RS:1;579b0f681375:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,238 INFO [RS:1;579b0f681375:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,37553,1733097383995-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T23:56:25,259 INFO [RS:0;579b0f681375:36617 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T23:56:25,262 INFO [RS:0;579b0f681375:36617 {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,36617,1733097383869-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,262 INFO [RS:0;579b0f681375:36617 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,262 INFO [RS:0;579b0f681375:36617 {}] regionserver.Replication(171): 579b0f681375,36617,1733097383869 started 2024-12-01T23:56:25,263 INFO [RS:1;579b0f681375:37553 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T23:56:25,263 INFO [RS:1;579b0f681375:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,37553,1733097383995-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,263 INFO [RS:1;579b0f681375:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,263 INFO [RS:1;579b0f681375:37553 {}] regionserver.Replication(171): 579b0f681375,37553,1733097383995 started 2024-12-01T23:56:25,281 INFO [RS:0;579b0f681375:36617 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,282 INFO [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer(1482): Serving as 579b0f681375,36617,1733097383869, RpcServer on 579b0f681375/172.17.0.2:36617, sessionid=0x101949cf63e0001 2024-12-01T23:56:25,283 DEBUG [RS:0;579b0f681375:36617 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T23:56:25,283 DEBUG [RS:0;579b0f681375:36617 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 579b0f681375,36617,1733097383869 2024-12-01T23:56:25,284 DEBUG [RS:0;579b0f681375:36617 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '579b0f681375,36617,1733097383869' 2024-12-01T23:56:25,284 DEBUG [RS:0;579b0f681375:36617 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T23:56:25,285 DEBUG [RS:0;579b0f681375:36617 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T23:56:25,285 DEBUG [RS:0;579b0f681375:36617 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T23:56:25,286 DEBUG [RS:0;579b0f681375:36617 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T23:56:25,286 DEBUG [RS:0;579b0f681375:36617 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 579b0f681375,36617,1733097383869 2024-12-01T23:56:25,286 DEBUG [RS:0;579b0f681375:36617 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '579b0f681375,36617,1733097383869' 2024-12-01T23:56:25,286 DEBUG [RS:0;579b0f681375:36617 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T23:56:25,287 DEBUG [RS:0;579b0f681375:36617 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T23:56:25,287 INFO [RS:1;579b0f681375:37553 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:25,287 INFO [RS:1;579b0f681375:37553 {}] regionserver.HRegionServer(1482): Serving as 579b0f681375,37553,1733097383995, RpcServer on 579b0f681375/172.17.0.2:37553, sessionid=0x101949cf63e0002 2024-12-01T23:56:25,287 DEBUG [RS:0;579b0f681375:36617 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T23:56:25,287 DEBUG [RS:1;579b0f681375:37553 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T23:56:25,287 DEBUG [RS:1;579b0f681375:37553 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 579b0f681375,37553,1733097383995 2024-12-01T23:56:25,287 INFO [RS:0;579b0f681375:36617 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T23:56:25,287 DEBUG [RS:1;579b0f681375:37553 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '579b0f681375,37553,1733097383995' 2024-12-01T23:56:25,287 INFO [RS:0;579b0f681375:36617 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T23:56:25,287 DEBUG [RS:1;579b0f681375:37553 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T23:56:25,288 DEBUG [RS:1;579b0f681375:37553 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T23:56:25,289 DEBUG [RS:1;579b0f681375:37553 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T23:56:25,289 DEBUG [RS:1;579b0f681375:37553 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T23:56:25,289 DEBUG [RS:1;579b0f681375:37553 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 579b0f681375,37553,1733097383995 2024-12-01T23:56:25,289 DEBUG [RS:1;579b0f681375:37553 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '579b0f681375,37553,1733097383995' 2024-12-01T23:56:25,289 DEBUG [RS:1;579b0f681375:37553 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T23:56:25,290 DEBUG [RS:1;579b0f681375:37553 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T23:56:25,290 DEBUG [RS:1;579b0f681375:37553 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T23:56:25,290 INFO [RS:1;579b0f681375:37553 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T23:56:25,290 INFO [RS:1;579b0f681375:37553 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T23:56:25,328 WARN [579b0f681375:35455 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T23:56:25,392 INFO [RS:0;579b0f681375:36617 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:56:25,392 INFO [RS:1;579b0f681375:37553 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:56:25,393 INFO [RS:0;579b0f681375:36617 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=579b0f681375%2C36617%2C1733097383869, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,36617,1733097383869, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/oldWALs, maxLogs=55 2024-12-01T23:56:25,393 INFO [RS:1;579b0f681375:37553 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=579b0f681375%2C37553%2C1733097383995, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,37553,1733097383995, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/oldWALs, maxLogs=55 2024-12-01T23:56:25,413 INFO [RS:0;579b0f681375:36617 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,36617,1733097383869/579b0f681375%2C36617%2C1733097383869.1733097385397 2024-12-01T23:56:25,413 INFO [RS:1;579b0f681375:37553 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,37553,1733097383995/579b0f681375%2C37553%2C1733097383995.1733097385397 2024-12-01T23:56:25,414 DEBUG [RS:1;579b0f681375:37553 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T23:56:25,414 DEBUG [RS:0;579b0f681375:36617 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T23:56:25,582 DEBUG [579b0f681375:35455 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-01T23:56:25,601 DEBUG [579b0f681375:35455 {}] balancer.BalancerClusterState(204): Hosts are {579b0f681375=0} racks are {/default-rack=0} 2024-12-01T23:56:25,604 DEBUG [579b0f681375:35455 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T23:56:25,604 DEBUG [579b0f681375:35455 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T23:56:25,604 DEBUG [579b0f681375:35455 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T23:56:25,604 DEBUG [579b0f681375:35455 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T23:56:25,604 INFO [579b0f681375:35455 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T23:56:25,604 INFO [579b0f681375:35455 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T23:56:25,605 DEBUG [579b0f681375:35455 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T23:56:25,614 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=579b0f681375,36617,1733097383869 2024-12-01T23:56:25,619 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 579b0f681375,36617,1733097383869, state=OPENING 2024-12-01T23:56:25,662 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-01T23:56:25,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:25,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:25,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:25,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:25,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:25,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:25,676 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T23:56:25,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=579b0f681375,36617,1733097383869}] 2024-12-01T23:56:25,859 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T23:56:25,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41263, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T23:56:25,869 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-01T23:56:25,869 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:56:25,870 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-01T23:56:25,871 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=579b0f681375%2C36617%2C1733097383869.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,36617,1733097383869, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/oldWALs, maxLogs=55 2024-12-01T23:56:25,890 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,36617,1733097383869/579b0f681375%2C36617%2C1733097383869.meta.1733097385873.meta 2024-12-01T23:56:25,890 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T23:56:25,891 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:56:25,892 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-01T23:56:25,895 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-01T23:56:25,899 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-01T23:56:25,903 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-01T23:56:25,903 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:56:25,904 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-01T23:56:25,904 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-01T23:56:25,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-01T23:56:25,906 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T23:56:25,906 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:25,906 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:25,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-01T23:56:25,907 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T23:56:25,907 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:25,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:25,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-01T23:56:25,908 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T23:56:25,908 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:25,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:25,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-01T23:56:25,909 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T23:56:25,909 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:25,910 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:25,910 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T23:56:25,911 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740 2024-12-01T23:56:25,911 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740 2024-12-01T23:56:25,912 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T23:56:25,912 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T23:56:25,912 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T23:56:25,913 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T23:56:25,913 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65440380, jitterRate=-0.024862349033355713}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T23:56:25,914 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-01T23:56:25,915 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733097385904Writing region info on filesystem at 1733097385904Initializing all the Stores at 1733097385905 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097385905Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097385905Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733097385905Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097385905Cleaning up temporary data from old regions at 1733097385912 (+7 ms)Running coprocessor post-open hooks at 1733097385914 (+2 ms)Region opened successfully at 1733097385915 (+1 ms) 2024-12-01T23:56:25,920 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733097385850 2024-12-01T23:56:25,929 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-01T23:56:25,930 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-01T23:56:25,931 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=579b0f681375,36617,1733097383869 2024-12-01T23:56:25,932 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 579b0f681375,36617,1733097383869, state=OPEN 2024-12-01T23:56:25,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:56:25,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:56:25,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:56:25,955 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:25,955 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:25,955 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:25,955 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=579b0f681375,36617,1733097383869 2024-12-01T23:56:25,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-01T23:56:25,961 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=579b0f681375,36617,1733097383869 in 277 msec 2024-12-01T23:56:25,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-01T23:56:25,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 799 msec 2024-12-01T23:56:25,968 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T23:56:25,968 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-01T23:56:25,985 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T23:56:25,986 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=579b0f681375,36617,1733097383869, seqNum=-1] 2024-12-01T23:56:26,003 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T23:56:26,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47889, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T23:56:26,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0660 sec 2024-12-01T23:56:26,021 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733097386021, completionTime=-1 2024-12-01T23:56:26,023 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-01T23:56:26,024 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-01T23:56:26,047 INFO [master/579b0f681375:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-01T23:56:26,048 INFO [master/579b0f681375:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733097446047 2024-12-01T23:56:26,048 INFO [master/579b0f681375:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733097506048 2024-12-01T23:56:26,048 INFO [master/579b0f681375:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-12-01T23:56:26,050 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-01T23:56:26,056 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,35455,1733097383050-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:26,056 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,35455,1733097383050-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:26,057 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,35455,1733097383050-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:26,058 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-579b0f681375:35455, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:26,059 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:26,059 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:26,066 DEBUG [master/579b0f681375:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-01T23:56:26,089 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.966sec 2024-12-01T23:56:26,091 INFO [master/579b0f681375:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-01T23:56:26,092 INFO [master/579b0f681375:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-01T23:56:26,093 INFO [master/579b0f681375:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-01T23:56:26,093 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-01T23:56:26,093 INFO [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-01T23:56:26,094 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,35455,1733097383050-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T23:56:26,094 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,35455,1733097383050-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-01T23:56:26,096 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-01T23:56:26,097 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-01T23:56:26,097 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,35455,1733097383050-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:26,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T23:56:26,168 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-01T23:56:26,168 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-01T23:56:26,172 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 579b0f681375,35455,-1 for getting cluster id 2024-12-01T23:56:26,175 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T23:56:26,182 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c7c053aa-d823-4a37-ab05-ab423c4adf45' 2024-12-01T23:56:26,184 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T23:56:26,184 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c7c053aa-d823-4a37-ab05-ab423c4adf45" 2024-12-01T23:56:26,187 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15372055, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T23:56:26,187 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [579b0f681375,35455,-1] 2024-12-01T23:56:26,189 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T23:56:26,191 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:26,192 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T23:56:26,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T23:56:26,195 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T23:56:26,201 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=579b0f681375,36617,1733097383869, seqNum=-1] 2024-12-01T23:56:26,202 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T23:56:26,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T23:56:26,238 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=579b0f681375,35455,1733097383050 2024-12-01T23:56:26,246 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-01T23:56:26,253 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 579b0f681375,35455,1733097383050 2024-12-01T23:56:26,255 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@73f2a1c7 2024-12-01T23:56:26,256 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-01T23:56:26,259 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-01T23:56:26,264 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35455 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733097386239', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T23:56:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35455 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733097386239 2024-12-01T23:56:26,271 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733097386239 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T23:56:26,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35455 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733097386239" procId is: 4 2024-12-01T23:56:26,273 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:26,274 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733097386239 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T23:56:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35455 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T23:56:26,287 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733097386239-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c0c2c2247dc5198133ebfb5bdcd7e7d0, NAME => 'testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733097386239', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357 2024-12-01T23:56:26,293 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733097386239-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:56:26,293 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733097386239-pool-0 {}] regionserver.HRegion(1722): Closing c0c2c2247dc5198133ebfb5bdcd7e7d0, disabling compactions & flushes 2024-12-01T23:56:26,293 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733097386239-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:26,293 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733097386239-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:26,293 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733097386239-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. after waiting 0 ms 2024-12-01T23:56:26,294 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733097386239-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:26,294 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733097386239-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:26,294 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733097386239-pool-0 {}] regionserver.HRegion(1676): Region close journal for c0c2c2247dc5198133ebfb5bdcd7e7d0: Waiting for close lock at 1733097386293Disabling compacts and flushes for region at 1733097386293Disabling writes for close at 1733097386293Writing region close event to WAL at 1733097386294 (+1 ms)Closed at 1733097386294 2024-12-01T23:56:26,295 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733097386239 execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T23:56:26,299 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733097386295"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733097386295"}]},"ts":"1733097386295"} 2024-12-01T23:56:26,303 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T23:56:26,304 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733097386239 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T23:56:26,306 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733097386239","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733097386304"}]},"ts":"1733097386304"} 2024-12-01T23:56:26,310 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733097386239, state=ENABLING in hbase:meta 2024-12-01T23:56:26,311 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {579b0f681375=0} racks are {/default-rack=0} 2024-12-01T23:56:26,311 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T23:56:26,311 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T23:56:26,311 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T23:56:26,311 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T23:56:26,311 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T23:56:26,311 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T23:56:26,311 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T23:56:26,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733097386239, region=c0c2c2247dc5198133ebfb5bdcd7e7d0, ASSIGN}] 2024-12-01T23:56:26,314 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733097386239, region=c0c2c2247dc5198133ebfb5bdcd7e7d0, ASSIGN 2024-12-01T23:56:26,315 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733097386239, region=c0c2c2247dc5198133ebfb5bdcd7e7d0, ASSIGN; state=OFFLINE, location=579b0f681375,37553,1733097383995; forceNewPlan=false, retain=false 2024-12-01T23:56:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35455 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T23:56:26,472 INFO [579b0f681375:35455 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-01T23:56:26,474 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c0c2c2247dc5198133ebfb5bdcd7e7d0, regionState=OPENING, regionLocation=579b0f681375,37553,1733097383995 2024-12-01T23:56:26,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733097386239, region=c0c2c2247dc5198133ebfb5bdcd7e7d0, ASSIGN because future has completed 2024-12-01T23:56:26,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0c2c2247dc5198133ebfb5bdcd7e7d0, server=579b0f681375,37553,1733097383995}] 2024-12-01T23:56:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35455 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T23:56:26,633 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T23:56:26,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44011, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T23:56:26,637 INFO [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:26,637 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c0c2c2247dc5198133ebfb5bdcd7e7d0, NAME => 'testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:56:26,638 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733097386239 c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:26,638 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:56:26,638 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:26,638 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:26,639 INFO [StoreOpener-c0c2c2247dc5198133ebfb5bdcd7e7d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:26,643 INFO [StoreOpener-c0c2c2247dc5198133ebfb5bdcd7e7d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0c2c2247dc5198133ebfb5bdcd7e7d0 columnFamilyName family 2024-12-01T23:56:26,644 DEBUG [StoreOpener-c0c2c2247dc5198133ebfb5bdcd7e7d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:26,644 INFO [StoreOpener-c0c2c2247dc5198133ebfb5bdcd7e7d0-1 {}] regionserver.HStore(327): Store=c0c2c2247dc5198133ebfb5bdcd7e7d0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:56:26,644 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:26,646 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:26,646 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:26,646 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:26,646 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:26,647 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:26,652 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:56:26,653 INFO [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c0c2c2247dc5198133ebfb5bdcd7e7d0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73804157, jitterRate=0.09976764023303986}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T23:56:26,653 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:26,655 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c0c2c2247dc5198133ebfb5bdcd7e7d0: Running coprocessor pre-open hook at 1733097386638Writing region info on filesystem at 1733097386638Initializing all the Stores at 1733097386639 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733097386639Cleaning up temporary data from old regions at 1733097386646 (+7 ms)Running coprocessor post-open hooks at 1733097386653 (+7 ms)Region opened successfully at 1733097386654 (+1 ms) 2024-12-01T23:56:26,656 INFO [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0., pid=6, masterSystemTime=1733097386633 2024-12-01T23:56:26,659 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:26,659 INFO [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:26,660 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c0c2c2247dc5198133ebfb5bdcd7e7d0, regionState=OPEN, openSeqNum=2, regionLocation=579b0f681375,37553,1733097383995 2024-12-01T23:56:26,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0c2c2247dc5198133ebfb5bdcd7e7d0, server=579b0f681375,37553,1733097383995 because future has completed 2024-12-01T23:56:26,666 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-01T23:56:26,667 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c0c2c2247dc5198133ebfb5bdcd7e7d0, server=579b0f681375,37553,1733097383995 in 184 msec 2024-12-01T23:56:26,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-01T23:56:26,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733097386239, region=c0c2c2247dc5198133ebfb5bdcd7e7d0, ASSIGN in 354 msec 2024-12-01T23:56:26,670 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733097386239 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T23:56:26,670 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733097386239","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733097386670"}]},"ts":"1733097386670"} 2024-12-01T23:56:26,673 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733097386239, state=ENABLED in hbase:meta 2024-12-01T23:56:26,674 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733097386239 execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T23:56:26,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733097386239 in 408 msec 2024-12-01T23:56:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35455 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T23:56:26,910 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733097386239 completed 2024-12-01T23:56:26,910 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733097386239 get assigned. Timeout = 60000ms 2024-12-01T23:56:26,914 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T23:56:26,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733097386239 assigned to meta. Checking AM states. 2024-12-01T23:56:26,922 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T23:56:26,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733097386239 assigned. 2024-12-01T23:56:26,932 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733097386239', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0., hostname=579b0f681375,37553,1733097383995, seqNum=2] 2024-12-01T23:56:26,934 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T23:56:26,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T23:56:26,945 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c0c2c2247dc5198133ebfb5bdcd7e7d0 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-01T23:56:26,966 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8b5d4221120a4118ab91b0081baa23f5 2024-12-01T23:56:26,997 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/.tmp/family/8b5d4221120a4118ab91b0081baa23f5 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-01T23:56:27,003 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/.tmp/family/8b5d4221120a4118ab91b0081baa23f5 2024-12-01T23:56:27,031 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/.tmp/family/8b5d4221120a4118ab91b0081baa23f5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/family/8b5d4221120a4118ab91b0081baa23f5 2024-12-01T23:56:27,032 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/family/8b5d4221120a4118ab91b0081baa23f5, delay=956 ms 2024-12-01T23:56:27,033 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/family/8b5d4221120a4118ab91b0081baa23f5, entries=2, sequenceid=6, filesize=4.9 K 2024-12-01T23:56:27,039 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for c0c2c2247dc5198133ebfb5bdcd7e7d0 in 93ms, sequenceid=6, compaction requested=false 2024-12-01T23:56:27,040 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733097386239' 2024-12-01T23:56:27,042 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c0c2c2247dc5198133ebfb5bdcd7e7d0: 2024-12-01T23:56:27,991 DEBUG [hfile-prefetch-1733097387032 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8b5d4221120a4118ab91b0081baa23f5_0. 2024-12-01T23:56:27,997 DEBUG [hfile-prefetch-1733097387032 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8b5d4221120a4118ab91b0081baa23f5_117. 2024-12-01T23:56:27,999 DEBUG [hfile-prefetch-1733097387032 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/family/8b5d4221120a4118ab91b0081baa23f5, with total blocks 2, and data blocks 1 2024-12-01T23:56:27,999 DEBUG [hfile-prefetch-1733097387032 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-01T23:56:28,000 DEBUG [hfile-prefetch-1733097387032 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/family/8b5d4221120a4118ab91b0081baa23f5, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-01T23:56:28,000 DEBUG [hfile-prefetch-1733097387032 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/family/8b5d4221120a4118ab91b0081baa23f5. Will sleep 100ms and try the verification again. 2024-12-01T23:56:28,060 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35455 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=c0c2c2247dc5198133ebfb5bdcd7e7d0, source=579b0f681375,37553,1733097383995, destination=579b0f681375,36617,1733097383869, warming up region on 579b0f681375,36617,1733097383869 2024-12-01T23:56:28,063 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35455 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=c0c2c2247dc5198133ebfb5bdcd7e7d0, source=579b0f681375,37553,1733097383995, destination=579b0f681375,36617,1733097383869, running balancer 2024-12-01T23:56:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35455 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733097386239, region=c0c2c2247dc5198133ebfb5bdcd7e7d0, REOPEN/MOVE 2024-12-01T23:56:28,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,064 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733097386239, region=c0c2c2247dc5198133ebfb5bdcd7e7d0, REOPEN/MOVE 2024-12-01T23:56:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] regionserver.HRegion(7855): Warmup {ENCODED => c0c2c2247dc5198133ebfb5bdcd7e7d0, NAME => 'testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:56:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:56:28,065 INFO [StoreOpener-c0c2c2247dc5198133ebfb5bdcd7e7d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:28,066 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c0c2c2247dc5198133ebfb5bdcd7e7d0, regionState=CLOSING, regionLocation=579b0f681375,37553,1733097383995 2024-12-01T23:56:28,066 INFO [StoreOpener-c0c2c2247dc5198133ebfb5bdcd7e7d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0c2c2247dc5198133ebfb5bdcd7e7d0 columnFamilyName family 2024-12-01T23:56:28,066 DEBUG [StoreOpener-c0c2c2247dc5198133ebfb5bdcd7e7d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:28,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733097386239, region=c0c2c2247dc5198133ebfb5bdcd7e7d0, REOPEN/MOVE because future has completed 2024-12-01T23:56:28,070 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-01T23:56:28,070 DEBUG [StoreOpener-c0c2c2247dc5198133ebfb5bdcd7e7d0-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/family/8b5d4221120a4118ab91b0081baa23f5 2024-12-01T23:56:28,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0c2c2247dc5198133ebfb5bdcd7e7d0, server=579b0f681375,37553,1733097383995}] 2024-12-01T23:56:28,070 INFO [StoreOpener-c0c2c2247dc5198133ebfb5bdcd7e7d0-1 {}] regionserver.HStore(327): Store=c0c2c2247dc5198133ebfb5bdcd7e7d0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:56:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] regionserver.HRegion(1722): Closing c0c2c2247dc5198133ebfb5bdcd7e7d0, disabling compactions & flushes 2024-12-01T23:56:28,071 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. after waiting 0 ms 2024-12-01T23:56:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,072 DEBUG [StoreFileCloser-c0c2c2247dc5198133ebfb5bdcd7e7d0-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/family/8b5d4221120a4118ab91b0081baa23f5 2024-12-01T23:56:28,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] regionserver.HRegion(1676): Region close journal for c0c2c2247dc5198133ebfb5bdcd7e7d0: Waiting for close lock at 1733097388071Disabling compacts and flushes for region at 1733097388071Disabling writes for close at 1733097388071Writing region close event to WAL at 1733097388073 (+2 ms)Closed at 1733097388073 2024-12-01T23:56:28,075 WARN [hfile-prefetch-1733097387032 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/family/8b5d4221120a4118ab91b0081baa23f5, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-01T23:56:28,076 DEBUG [hfile-prefetch-1733097387032 {}] hfile.PrefetchExecutor(121): Prefetch completed for 8b5d4221120a4118ab91b0081baa23f5 2024-12-01T23:56:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35455 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-01T23:56:28,138 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-01T23:56:28,138 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-01T23:56:28,138 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T23:56:28,139 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T23:56:28,159 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-01T23:56:28,159 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733097388138 2024-12-01T23:56:28,159 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-01T23:56:28,230 INFO [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:28,231 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-01T23:56:28,233 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing c0c2c2247dc5198133ebfb5bdcd7e7d0, disabling compactions & flushes 2024-12-01T23:56:28,233 INFO [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,233 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,233 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. after waiting 0 ms 2024-12-01T23:56:28,233 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,237 DEBUG [StoreFileCloser-c0c2c2247dc5198133ebfb5bdcd7e7d0-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 8b5d4221120a4118ab91b0081baa23f5, starting offset: 0, end offset: 9223372036854775807 2024-12-01T23:56:28,240 DEBUG [StoreFileCloser-c0c2c2247dc5198133ebfb5bdcd7e7d0-family-1 {}] bucket.BucketCache(856): removed key 8b5d4221120a4118ab91b0081baa23f5_0 from back map with offset lock 2094080 in the evict process 2024-12-01T23:56:28,245 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-01T23:56:28,247 INFO [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,247 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for c0c2c2247dc5198133ebfb5bdcd7e7d0: Waiting for close lock at 1733097388233Running coprocessor pre-close hooks at 1733097388233Disabling compacts and flushes for region at 1733097388233Disabling writes for close at 1733097388233Writing region close event to WAL at 1733097388241 (+8 ms)Running coprocessor post-close hooks at 1733097388246 (+5 ms)Closed at 1733097388247 (+1 ms) 2024-12-01T23:56:28,247 INFO [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding c0c2c2247dc5198133ebfb5bdcd7e7d0 move to 579b0f681375,36617,1733097383869 record at close sequenceid=6 2024-12-01T23:56:28,250 INFO [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:28,251 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c0c2c2247dc5198133ebfb5bdcd7e7d0, regionState=CLOSED 2024-12-01T23:56:28,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0c2c2247dc5198133ebfb5bdcd7e7d0, server=579b0f681375,37553,1733097383995 because future has completed 2024-12-01T23:56:28,259 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-01T23:56:28,259 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-01T23:56:28,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-01T23:56:28,260 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T23:56:28,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure c0c2c2247dc5198133ebfb5bdcd7e7d0, server=579b0f681375,37553,1733097383995 in 187 msec 2024-12-01T23:56:28,260 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T23:56:28,261 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733097386239, region=c0c2c2247dc5198133ebfb5bdcd7e7d0, REOPEN/MOVE; state=CLOSED, location=579b0f681375,36617,1733097383869; forceNewPlan=false, retain=false 2024-12-01T23:56:28,261 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-01T23:56:28,261 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-01T23:56:28,261 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T23:56:28,261 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T23:56:28,263 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-01T23:56:28,263 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733097388259 2024-12-01T23:56:28,263 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-01T23:56:28,264 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-01T23:56:28,264 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733097388261 2024-12-01T23:56:28,264 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-01T23:56:28,411 INFO [579b0f681375:35455 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-01T23:56:28,412 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c0c2c2247dc5198133ebfb5bdcd7e7d0, regionState=OPENING, regionLocation=579b0f681375,36617,1733097383869 2024-12-01T23:56:28,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733097386239, region=c0c2c2247dc5198133ebfb5bdcd7e7d0, REOPEN/MOVE because future has completed 2024-12-01T23:56:28,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0c2c2247dc5198133ebfb5bdcd7e7d0, server=579b0f681375,36617,1733097383869}] 2024-12-01T23:56:28,575 INFO [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,575 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => c0c2c2247dc5198133ebfb5bdcd7e7d0, NAME => 'testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:56:28,576 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733097386239 c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:28,576 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:56:28,576 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:28,576 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:28,578 INFO [StoreOpener-c0c2c2247dc5198133ebfb5bdcd7e7d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:28,579 INFO [StoreOpener-c0c2c2247dc5198133ebfb5bdcd7e7d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0c2c2247dc5198133ebfb5bdcd7e7d0 columnFamilyName family 2024-12-01T23:56:28,579 DEBUG [StoreOpener-c0c2c2247dc5198133ebfb5bdcd7e7d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:28,580 DEBUG [StoreFileOpener-c0c2c2247dc5198133ebfb5bdcd7e7d0-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/family/8b5d4221120a4118ab91b0081baa23f5, delay=982 ms 2024-12-01T23:56:28,582 DEBUG [StoreOpener-c0c2c2247dc5198133ebfb5bdcd7e7d0-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/family/8b5d4221120a4118ab91b0081baa23f5 2024-12-01T23:56:28,582 INFO [StoreOpener-c0c2c2247dc5198133ebfb5bdcd7e7d0-1 {}] regionserver.HStore(327): Store=c0c2c2247dc5198133ebfb5bdcd7e7d0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:56:28,582 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:28,583 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:28,584 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:28,584 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:28,584 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:28,588 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:28,588 INFO [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened c0c2c2247dc5198133ebfb5bdcd7e7d0; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60472330, jitterRate=-0.09889206290245056}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T23:56:28,588 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:28,589 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for c0c2c2247dc5198133ebfb5bdcd7e7d0: Running coprocessor pre-open hook at 1733097388577Writing region info on filesystem at 1733097388577Initializing all the Stores at 1733097388577Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733097388577Cleaning up temporary data from old regions at 1733097388584 (+7 ms)Running coprocessor post-open hooks at 1733097388588 (+4 ms)Region opened successfully at 1733097388589 (+1 ms) 2024-12-01T23:56:28,590 INFO [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0., pid=9, masterSystemTime=1733097388570 2024-12-01T23:56:28,594 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,594 INFO [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,594 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c0c2c2247dc5198133ebfb5bdcd7e7d0, regionState=OPEN, openSeqNum=10, regionLocation=579b0f681375,36617,1733097383869 2024-12-01T23:56:28,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0c2c2247dc5198133ebfb5bdcd7e7d0, server=579b0f681375,36617,1733097383869 because future has completed 2024-12-01T23:56:28,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-01T23:56:28,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure c0c2c2247dc5198133ebfb5bdcd7e7d0, server=579b0f681375,36617,1733097383869 in 180 msec 2024-12-01T23:56:28,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733097386239, region=c0c2c2247dc5198133ebfb5bdcd7e7d0, REOPEN/MOVE in 538 msec 2024-12-01T23:56:28,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-01T23:56:28,617 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T23:56:28,617 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T23:56:28,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:28,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:28,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T23:56:28,622 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-01T23:56:28,622 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2143720667, stopped=false 2024-12-01T23:56:28,623 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=579b0f681375,35455,1733097383050 2024-12-01T23:56:28,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T23:56:28,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T23:56:28,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T23:56:28,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:28,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:28,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:28,654 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T23:56:28,655 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:56:28,655 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:56:28,656 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T23:56:28,656 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:56:28,656 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T23:56:28,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:28,657 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '579b0f681375,36617,1733097383869' ***** 2024-12-01T23:56:28,658 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-01T23:56:28,658 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '579b0f681375,37553,1733097383995' ***** 2024-12-01T23:56:28,658 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-01T23:56:28,658 INFO [RS:0;579b0f681375:36617 {}] bucket.BucketCache(1738): Disabling cache 2024-12-01T23:56:28,658 INFO [RS:1;579b0f681375:37553 {}] bucket.BucketCache(1738): Disabling cache 2024-12-01T23:56:28,659 INFO [RS:1;579b0f681375:37553 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/bucket.persistence 2024-12-01T23:56:28,659 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-01T23:56:28,659 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-01T23:56:28,659 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-01T23:56:28,659 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-01T23:56:28,659 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-01T23:56:28,659 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-01T23:56:28,659 INFO [RS:1;579b0f681375:37553 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-01T23:56:28,659 INFO [RS:0;579b0f681375:36617 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/bucket.persistence 2024-12-01T23:56:28,660 INFO [RS:0;579b0f681375:36617 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-01T23:56:28,659 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-01T23:56:28,660 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-01T23:56:28,669 DEBUG [RS:1;579b0f681375:37553 {}] bucket.BucketCache(1487): Thread RS:1;579b0f681375:37553 started persisting bucket cache to file 2024-12-01T23:56:28,670 DEBUG [RS:1;579b0f681375:37553 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T23:56:28,670 DEBUG [RS:0;579b0f681375:36617 {}] bucket.BucketCache(1487): Thread RS:0;579b0f681375:36617 started persisting bucket cache to file 2024-12-01T23:56:28,670 DEBUG [RS:1;579b0f681375:37553 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T23:56:28,670 DEBUG [RS:0;579b0f681375:36617 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T23:56:28,670 DEBUG [RS:0;579b0f681375:36617 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T23:56:28,673 DEBUG [RS:0;579b0f681375:36617 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-01T23:56:28,673 DEBUG [RS:1;579b0f681375:37553 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-01T23:56:28,673 DEBUG [RS:0;579b0f681375:36617 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733097388670 2024-12-01T23:56:28,673 DEBUG [RS:1;579b0f681375:37553 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733097388670 2024-12-01T23:56:28,673 DEBUG [RS:1;579b0f681375:37553 {}] bucket.BucketCache(1509): Thread RS:1;579b0f681375:37553 finished persisting bucket cache to file, renaming 2024-12-01T23:56:28,673 INFO [RS:1;579b0f681375:37553 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T23:56:28,674 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T23:56:28,674 INFO [RS:1;579b0f681375:37553 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T23:56:28,674 INFO [RS:1;579b0f681375:37553 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T23:56:28,674 INFO [RS:1;579b0f681375:37553 {}] regionserver.HRegionServer(959): stopping server 579b0f681375,37553,1733097383995 2024-12-01T23:56:28,674 INFO [RS:1;579b0f681375:37553 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T23:56:28,674 INFO [RS:1;579b0f681375:37553 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;579b0f681375:37553. 2024-12-01T23:56:28,674 DEBUG [RS:1;579b0f681375:37553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T23:56:28,674 DEBUG [RS:1;579b0f681375:37553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:28,675 INFO [RS:1;579b0f681375:37553 {}] regionserver.HRegionServer(976): stopping server 579b0f681375,37553,1733097383995; all regions closed. 2024-12-01T23:56:28,679 DEBUG [RS:1;579b0f681375:37553 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/oldWALs 2024-12-01T23:56:28,679 INFO [RS:1;579b0f681375:37553 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 579b0f681375%2C37553%2C1733097383995:(num 1733097385397) 2024-12-01T23:56:28,679 DEBUG [RS:1;579b0f681375:37553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:28,679 INFO [RS:1;579b0f681375:37553 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T23:56:28,680 INFO [RS:1;579b0f681375:37553 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T23:56:28,680 INFO [RS:1;579b0f681375:37553 {}] hbase.ChoreService(370): Chore service for: regionserver/579b0f681375:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-01T23:56:28,680 INFO [RS:1;579b0f681375:37553 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T23:56:28,680 INFO [regionserver/579b0f681375:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T23:56:28,680 INFO [RS:1;579b0f681375:37553 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T23:56:28,680 INFO [RS:1;579b0f681375:37553 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T23:56:28,680 INFO [RS:1;579b0f681375:37553 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T23:56:28,681 INFO [RS:1;579b0f681375:37553 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37553 2024-12-01T23:56:28,690 DEBUG [RS:0;579b0f681375:36617 {}] bucket.BucketCache(1509): Thread RS:0;579b0f681375:36617 finished persisting bucket cache to file, renaming 2024-12-01T23:56:28,691 WARN [RS:0;579b0f681375:36617 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-01T23:56:28,691 INFO [RS:0;579b0f681375:36617 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T23:56:28,691 INFO [RS:0;579b0f681375:36617 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T23:56:28,691 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T23:56:28,691 INFO [RS:0;579b0f681375:36617 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T23:56:28,691 INFO [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer(3091): Received CLOSE for c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:28,692 INFO [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer(959): stopping server 579b0f681375,36617,1733097383869 2024-12-01T23:56:28,692 INFO [RS:0;579b0f681375:36617 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T23:56:28,692 INFO [RS:0;579b0f681375:36617 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;579b0f681375:36617. 2024-12-01T23:56:28,692 DEBUG [RS:0;579b0f681375:36617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T23:56:28,692 DEBUG [RS:0;579b0f681375:36617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:28,692 INFO [RS:0;579b0f681375:36617 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T23:56:28,692 INFO [RS:0;579b0f681375:36617 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T23:56:28,692 INFO [RS:0;579b0f681375:36617 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T23:56:28,692 INFO [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-01T23:56:28,692 INFO [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-01T23:56:28,692 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c0c2c2247dc5198133ebfb5bdcd7e7d0, disabling compactions & flushes 2024-12-01T23:56:28,692 INFO [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,693 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,693 INFO [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-01T23:56:28,693 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. after waiting 0 ms 2024-12-01T23:56:28,693 DEBUG [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, c0c2c2247dc5198133ebfb5bdcd7e7d0=testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0.} 2024-12-01T23:56:28,693 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,693 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-01T23:56:28,693 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T23:56:28,693 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T23:56:28,693 DEBUG [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c0c2c2247dc5198133ebfb5bdcd7e7d0 2024-12-01T23:56:28,693 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T23:56:28,693 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T23:56:28,693 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T23:56:28,693 DEBUG [StoreFileCloser-c0c2c2247dc5198133ebfb5bdcd7e7d0-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/family/8b5d4221120a4118ab91b0081baa23f5 2024-12-01T23:56:28,694 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-01T23:56:28,698 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnRegionMove1733097386239/c0c2c2247dc5198133ebfb5bdcd7e7d0/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-01T23:56:28,698 INFO [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,698 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c0c2c2247dc5198133ebfb5bdcd7e7d0: Waiting for close lock at 1733097388692Running coprocessor pre-close hooks at 1733097388692Disabling compacts and flushes for region at 1733097388692Disabling writes for close at 1733097388693 (+1 ms)Writing region close event to WAL at 1733097388694 (+1 ms)Running coprocessor post-close hooks at 1733097388698 (+4 ms)Closed at 1733097388698 2024-12-01T23:56:28,699 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0. 2024-12-01T23:56:28,702 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7512666465444e09ae90d9060c2877be 2024-12-01T23:56:28,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/579b0f681375,37553,1733097383995 2024-12-01T23:56:28,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T23:56:28,717 INFO [RS:1;579b0f681375:37553 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T23:56:28,717 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/info/7512666465444e09ae90d9060c2877be is 205, key is testBlockEvictionOnRegionMove1733097386239,,1733097386260.c0c2c2247dc5198133ebfb5bdcd7e7d0./info:regioninfo/1733097388594/Put/seqid=0 2024-12-01T23:56:28,718 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/info/7512666465444e09ae90d9060c2877be 2024-12-01T23:56:28,726 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 641381bbc5b5477f913c106d864d6e55 2024-12-01T23:56:28,729 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [579b0f681375,37553,1733097383995] 2024-12-01T23:56:28,737 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/579b0f681375,37553,1733097383995 already deleted, retry=false 2024-12-01T23:56:28,737 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/ns/641381bbc5b5477f913c106d864d6e55 is 43, key is default/ns:d/1733097386009/Put/seqid=0 2024-12-01T23:56:28,737 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 579b0f681375,37553,1733097383995 expired; onlineServers=1 2024-12-01T23:56:28,737 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/ns/641381bbc5b5477f913c106d864d6e55 2024-12-01T23:56:28,742 INFO [regionserver/579b0f681375:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T23:56:28,743 INFO [regionserver/579b0f681375:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T23:56:28,745 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c521aaa07c4b42ff906bae03dd94e03a 2024-12-01T23:56:28,756 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/table/c521aaa07c4b42ff906bae03dd94e03a is 78, key is testBlockEvictionOnRegionMove1733097386239/table:state/1733097386670/Put/seqid=0 2024-12-01T23:56:28,757 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/table/c521aaa07c4b42ff906bae03dd94e03a 2024-12-01T23:56:28,759 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/info/7512666465444e09ae90d9060c2877be as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/info/7512666465444e09ae90d9060c2877be 2024-12-01T23:56:28,761 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/info/7512666465444e09ae90d9060c2877be, delay=969 ms 2024-12-01T23:56:28,762 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/info/7512666465444e09ae90d9060c2877be, entries=15, sequenceid=15, filesize=7.6 K 2024-12-01T23:56:28,762 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/ns/641381bbc5b5477f913c106d864d6e55 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/ns/641381bbc5b5477f913c106d864d6e55 2024-12-01T23:56:28,763 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/ns/641381bbc5b5477f913c106d864d6e55, delay=988 ms 2024-12-01T23:56:28,765 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/ns/641381bbc5b5477f913c106d864d6e55, entries=2, sequenceid=15, filesize=5.0 K 2024-12-01T23:56:28,765 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/table/c521aaa07c4b42ff906bae03dd94e03a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/table/c521aaa07c4b42ff906bae03dd94e03a 2024-12-01T23:56:28,766 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/table/c521aaa07c4b42ff906bae03dd94e03a, delay=941 ms 2024-12-01T23:56:28,767 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/table/c521aaa07c4b42ff906bae03dd94e03a, entries=2, sequenceid=15, filesize=5.3 K 2024-12-01T23:56:28,768 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=15, compaction requested=false 2024-12-01T23:56:28,768 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-01T23:56:28,769 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/info/7512666465444e09ae90d9060c2877be 2024-12-01T23:56:28,770 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/ns/641381bbc5b5477f913c106d864d6e55 2024-12-01T23:56:28,771 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/table/c521aaa07c4b42ff906bae03dd94e03a 2024-12-01T23:56:28,775 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-01T23:56:28,776 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T23:56:28,776 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T23:56:28,777 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733097388693Running coprocessor pre-close hooks at 1733097388693Disabling compacts and flushes for region at 1733097388693Disabling writes for close at 1733097388693Obtaining lock to block concurrent updates at 1733097388694 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733097388694Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733097388694Flushing stores of hbase:meta,,1.1588230740 at 1733097388695 (+1 ms)Flushing 1588230740/info: creating writer at 1733097388695Flushing 1588230740/info: appending metadata at 1733097388716 (+21 ms)Flushing 1588230740/info: closing flushed file at 1733097388716Flushing 1588230740/ns: creating writer at 1733097388721 (+5 ms)Flushing 1588230740/ns: appending metadata at 1733097388736 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733097388736Flushing 1588230740/table: creating writer at 1733097388739 (+3 ms)Flushing 1588230740/table: appending metadata at 1733097388756 (+17 ms)Flushing 1588230740/table: closing flushed file at 1733097388756Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@198f7f83: reopening flushed file at 1733097388759 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ba492ea: reopening flushed file at 1733097388762 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@494249d3: reopening flushed file at 1733097388765 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=15, compaction requested=false at 1733097388768 (+3 ms)Writing region close event to WAL at 1733097388771 (+3 ms)Running coprocessor post-close hooks at 1733097388776 (+5 ms)Closed at 1733097388776 2024-12-01T23:56:28,777 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-01T23:56:28,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:56:28,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37553-0x101949cf63e0002, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:56:28,830 INFO [RS:1;579b0f681375:37553 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T23:56:28,830 INFO [RS:1;579b0f681375:37553 {}] regionserver.HRegionServer(1031): Exiting; stopping=579b0f681375,37553,1733097383995; zookeeper connection closed. 2024-12-01T23:56:28,830 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@ddb8fe6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@ddb8fe6 2024-12-01T23:56:28,893 INFO [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer(976): stopping server 579b0f681375,36617,1733097383869; all regions closed. 2024-12-01T23:56:28,898 DEBUG [RS:0;579b0f681375:36617 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/oldWALs 2024-12-01T23:56:28,898 INFO [RS:0;579b0f681375:36617 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 579b0f681375%2C36617%2C1733097383869.meta:.meta(num 1733097385873) 2024-12-01T23:56:28,903 DEBUG [RS:0;579b0f681375:36617 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/oldWALs 2024-12-01T23:56:28,903 INFO [RS:0;579b0f681375:36617 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 579b0f681375%2C36617%2C1733097383869:(num 1733097385397) 2024-12-01T23:56:28,903 DEBUG [RS:0;579b0f681375:36617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:28,903 INFO [RS:0;579b0f681375:36617 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T23:56:28,903 INFO [RS:0;579b0f681375:36617 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T23:56:28,904 INFO [RS:0;579b0f681375:36617 {}] hbase.ChoreService(370): Chore service for: regionserver/579b0f681375:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-01T23:56:28,904 INFO [RS:0;579b0f681375:36617 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T23:56:28,905 INFO [regionserver/579b0f681375:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T23:56:28,905 INFO [RS:0;579b0f681375:36617 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36617 2024-12-01T23:56:28,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/579b0f681375,36617,1733097383869 2024-12-01T23:56:28,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T23:56:28,929 INFO [RS:0;579b0f681375:36617 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T23:56:28,929 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$249/0x00007f808847c228@29d13b51 rejected from java.util.concurrent.ThreadPoolExecutor@6cd3a00f[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 14] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-01T23:56:28,937 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [579b0f681375,36617,1733097383869] 2024-12-01T23:56:28,945 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/579b0f681375,36617,1733097383869 already deleted, retry=false 2024-12-01T23:56:28,945 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 579b0f681375,36617,1733097383869 expired; onlineServers=0 2024-12-01T23:56:28,945 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '579b0f681375,35455,1733097383050' ***** 2024-12-01T23:56:28,945 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-01T23:56:28,945 INFO [M:0;579b0f681375:35455 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T23:56:28,946 INFO [M:0;579b0f681375:35455 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T23:56:28,946 DEBUG [M:0;579b0f681375:35455 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-01T23:56:28,946 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-01T23:56:28,946 DEBUG [M:0;579b0f681375:35455 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-01T23:56:28,946 DEBUG [master/579b0f681375:0:becomeActiveMaster-HFileCleaner.large.0-1733097385081 {}] cleaner.HFileCleaner(306): Exit Thread[master/579b0f681375:0:becomeActiveMaster-HFileCleaner.large.0-1733097385081,5,FailOnTimeoutGroup] 2024-12-01T23:56:28,946 DEBUG [master/579b0f681375:0:becomeActiveMaster-HFileCleaner.small.0-1733097385085 {}] cleaner.HFileCleaner(306): Exit Thread[master/579b0f681375:0:becomeActiveMaster-HFileCleaner.small.0-1733097385085,5,FailOnTimeoutGroup] 2024-12-01T23:56:28,946 INFO [M:0;579b0f681375:35455 {}] hbase.ChoreService(370): Chore service for: master/579b0f681375:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-01T23:56:28,946 INFO [M:0;579b0f681375:35455 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T23:56:28,947 DEBUG [M:0;579b0f681375:35455 {}] master.HMaster(1795): Stopping service threads 2024-12-01T23:56:28,947 INFO [M:0;579b0f681375:35455 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-01T23:56:28,947 INFO [M:0;579b0f681375:35455 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T23:56:28,948 INFO [M:0;579b0f681375:35455 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-01T23:56:28,948 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-01T23:56:28,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-01T23:56:28,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:28,954 DEBUG [M:0;579b0f681375:35455 {}] zookeeper.ZKUtil(347): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-01T23:56:28,954 WARN [M:0;579b0f681375:35455 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-01T23:56:28,955 INFO [M:0;579b0f681375:35455 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/.lastflushedseqids 2024-12-01T23:56:28,965 INFO [M:0;579b0f681375:35455 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-01T23:56:28,965 INFO [M:0;579b0f681375:35455 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-01T23:56:28,965 DEBUG [M:0;579b0f681375:35455 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T23:56:28,965 INFO [M:0;579b0f681375:35455 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:56:28,965 DEBUG [M:0;579b0f681375:35455 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:56:28,965 DEBUG [M:0;579b0f681375:35455 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T23:56:28,966 DEBUG [M:0;579b0f681375:35455 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:56:28,966 INFO [M:0;579b0f681375:35455 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-12-01T23:56:28,975 DEBUG [M:0;579b0f681375:35455 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0ec5721b668c48b8a367f48bb2365cef 2024-12-01T23:56:28,988 DEBUG [M:0;579b0f681375:35455 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0ec5721b668c48b8a367f48bb2365cef is 82, key is hbase:meta,,1/info:regioninfo/1733097385930/Put/seqid=0 2024-12-01T23:56:28,988 INFO [M:0;579b0f681375:35455 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0ec5721b668c48b8a367f48bb2365cef 2024-12-01T23:56:28,995 DEBUG [M:0;579b0f681375:35455 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6264d4b6e1754a54a2a9607c55009c8d 2024-12-01T23:56:29,007 DEBUG [M:0;579b0f681375:35455 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6264d4b6e1754a54a2a9607c55009c8d is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733097386675/Put/seqid=0 2024-12-01T23:56:29,008 INFO [M:0;579b0f681375:35455 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6264d4b6e1754a54a2a9607c55009c8d 2024-12-01T23:56:29,015 DEBUG [M:0;579b0f681375:35455 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 827a903450ce468baee76d137bea425d 2024-12-01T23:56:29,027 DEBUG [M:0;579b0f681375:35455 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/827a903450ce468baee76d137bea425d is 69, key is 579b0f681375,36617,1733097383869/rs:state/1733097385101/Put/seqid=0 2024-12-01T23:56:29,028 INFO [M:0;579b0f681375:35455 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/827a903450ce468baee76d137bea425d 2024-12-01T23:56:29,029 DEBUG [M:0;579b0f681375:35455 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0ec5721b668c48b8a367f48bb2365cef as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0ec5721b668c48b8a367f48bb2365cef 2024-12-01T23:56:29,031 INFO [M:0;579b0f681375:35455 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0ec5721b668c48b8a367f48bb2365cef, entries=8, sequenceid=84, filesize=5.5 K 2024-12-01T23:56:29,031 DEBUG [M:0;579b0f681375:35455 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6264d4b6e1754a54a2a9607c55009c8d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6264d4b6e1754a54a2a9607c55009c8d 2024-12-01T23:56:29,033 INFO [M:0;579b0f681375:35455 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6264d4b6e1754a54a2a9607c55009c8d, entries=9, sequenceid=84, filesize=6.5 K 2024-12-01T23:56:29,033 DEBUG [M:0;579b0f681375:35455 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/827a903450ce468baee76d137bea425d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/827a903450ce468baee76d137bea425d 2024-12-01T23:56:29,034 INFO [M:0;579b0f681375:35455 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/827a903450ce468baee76d137bea425d, entries=2, sequenceid=84, filesize=5.1 K 2024-12-01T23:56:29,035 INFO [M:0;579b0f681375:35455 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 69ms, sequenceid=84, compaction requested=false 2024-12-01T23:56:29,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:56:29,037 INFO [RS:0;579b0f681375:36617 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T23:56:29,037 INFO [RS:0;579b0f681375:36617 {}] regionserver.HRegionServer(1031): Exiting; stopping=579b0f681375,36617,1733097383869; zookeeper connection closed. 2024-12-01T23:56:29,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36617-0x101949cf63e0001, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:56:29,037 INFO [M:0;579b0f681375:35455 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:56:29,037 DEBUG [M:0;579b0f681375:35455 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733097388965Disabling compacts and flushes for region at 1733097388965Disabling writes for close at 1733097388965Obtaining lock to block concurrent updates at 1733097388966 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733097388966Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1733097388966Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733097388967 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733097388967Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733097388987 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733097388987Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733097388990 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733097389007 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733097389007Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733097389010 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733097389026 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733097389026Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37b87542: reopening flushed file at 1733097389029 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d45f28b: reopening flushed file at 1733097389031 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33c6155: reopening flushed file at 1733097389033 (+2 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 69ms, sequenceid=84, compaction requested=false at 1733097389035 (+2 ms)Writing region close event to WAL at 1733097389037 (+2 ms)Closed at 1733097389037 2024-12-01T23:56:29,037 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2202594c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2202594c 2024-12-01T23:56:29,038 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-01T23:56:29,039 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T23:56:29,039 INFO [M:0;579b0f681375:35455 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-01T23:56:29,039 INFO [M:0;579b0f681375:35455 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35455 2024-12-01T23:56:29,039 INFO [M:0;579b0f681375:35455 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T23:56:29,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:56:29,162 INFO [M:0;579b0f681375:35455 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T23:56:29,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35455-0x101949cf63e0000, quorum=127.0.0.1:49794, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:56:29,170 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-01T23:56:29,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-01T23:56:29,190 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:29,197 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/579b0f681375:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/579b0f681375:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733097388761 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733097388580 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/579b0f681375:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733097387032 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/579b0f681375:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733097388763 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=358 (was 389), ProcessCount=11 (was 11), AvailableMemoryMB=7531 (was 7899) 2024-12-01T23:56:29,202 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=358, ProcessCount=11, AvailableMemoryMB=7531 2024-12-01T23:56:29,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/hadoop-log-dir so I do NOT create it in target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66 2024-12-01T23:56:29,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/hadoop-tmp-dir so I do NOT create it in target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66 2024-12-01T23:56:29,202 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:29,205 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/cluster_351da07b-ddd9-9307-f64a-d0033bd39c01, deleteOnExit=true 2024-12-01T23:56:29,209 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/cluster_351da07b-ddd9-9307-f64a-d0033bd39c01/zookeeper_0, clientPort=56584, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/cluster_351da07b-ddd9-9307-f64a-d0033bd39c01/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/cluster_351da07b-ddd9-9307-f64a-d0033bd39c01/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-01T23:56:29,210 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56584 2024-12-01T23:56:29,224 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357 with version=8 2024-12-01T23:56:29,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/hbase-staging 2024-12-01T23:56:29,226 INFO [Time-limited test {}] client.ConnectionUtils(128): master/579b0f681375:0 server-side Connection retries=45 2024-12-01T23:56:29,226 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:29,227 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:29,227 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T23:56:29,227 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:29,227 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T23:56:29,227 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-01T23:56:29,227 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T23:56:29,228 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35987 2024-12-01T23:56:29,229 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35987 connecting to ZooKeeper ensemble=127.0.0.1:56584 2024-12-01T23:56:29,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:359870x0, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T23:56:29,272 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35987-0x101949d0f5d0000 connected 2024-12-01T23:56:29,338 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:29,339 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:29,341 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:56:29,341 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357, hbase.cluster.distributed=false 2024-12-01T23:56:29,344 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T23:56:29,345 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35987 2024-12-01T23:56:29,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35987 2024-12-01T23:56:29,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35987 2024-12-01T23:56:29,347 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35987 2024-12-01T23:56:29,347 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35987 2024-12-01T23:56:29,363 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/579b0f681375:0 server-side Connection retries=45 2024-12-01T23:56:29,363 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:29,363 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:29,363 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T23:56:29,363 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:29,364 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T23:56:29,364 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T23:56:29,364 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T23:56:29,364 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45041 2024-12-01T23:56:29,365 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45041 connecting to ZooKeeper ensemble=127.0.0.1:56584 2024-12-01T23:56:29,366 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:29,366 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:29,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:450410x0, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T23:56:29,379 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45041-0x101949d0f5d0001 connected 2024-12-01T23:56:29,380 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:56:29,380 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T23:56:29,381 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/bucket.cache 2024-12-01T23:56:29,381 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-01T23:56:29,381 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-01T23:56:29,381 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-01T23:56:29,381 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-01T23:56:29,381 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-01T23:56:29,381 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-01T23:56:29,382 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T23:56:29,382 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-01T23:56:29,382 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/bucket.persistence is complete. 2024-12-01T23:56:29,382 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T23:56:29,384 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T23:56:29,385 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45041 2024-12-01T23:56:29,385 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45041 2024-12-01T23:56:29,385 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45041 2024-12-01T23:56:29,386 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45041 2024-12-01T23:56:29,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45041 2024-12-01T23:56:29,400 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/579b0f681375:0 server-side Connection retries=45 2024-12-01T23:56:29,400 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:29,400 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:29,400 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T23:56:29,400 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:29,401 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T23:56:29,401 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T23:56:29,401 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T23:56:29,401 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45601 2024-12-01T23:56:29,403 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45601 connecting to ZooKeeper ensemble=127.0.0.1:56584 2024-12-01T23:56:29,403 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:29,403 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:29,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:456010x0, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T23:56:29,413 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45601-0x101949d0f5d0002 connected 2024-12-01T23:56:29,413 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:56:29,413 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T23:56:29,414 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/bucket.cache 2024-12-01T23:56:29,414 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-01T23:56:29,414 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-01T23:56:29,414 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-01T23:56:29,415 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-01T23:56:29,415 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-01T23:56:29,415 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-01T23:56:29,415 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T23:56:29,415 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-01T23:56:29,415 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/bucket.persistence is complete. 2024-12-01T23:56:29,416 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T23:56:29,417 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T23:56:29,418 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45601 2024-12-01T23:56:29,418 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45601 2024-12-01T23:56:29,418 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45601 2024-12-01T23:56:29,422 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45601 2024-12-01T23:56:29,422 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45601 2024-12-01T23:56:29,433 DEBUG [M:0;579b0f681375:35987 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;579b0f681375:35987 2024-12-01T23:56:29,433 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/579b0f681375,35987,1733097389226 2024-12-01T23:56:29,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:56:29,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:56:29,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:56:29,438 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/579b0f681375,35987,1733097389226 2024-12-01T23:56:29,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T23:56:29,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T23:56:29,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:29,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:29,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:29,447 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-01T23:56:29,447 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/579b0f681375,35987,1733097389226 from backup master directory 2024-12-01T23:56:29,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/579b0f681375,35987,1733097389226 2024-12-01T23:56:29,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:56:29,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:56:29,454 WARN [master/579b0f681375:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T23:56:29,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:56:29,454 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=579b0f681375,35987,1733097389226 2024-12-01T23:56:29,456 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/hbase.id] with ID: cdd43f0f-e0bf-4fa4-8762-990849e15fef 2024-12-01T23:56:29,456 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/.tmp/hbase.id 2024-12-01T23:56:29,460 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/hbase.id] 2024-12-01T23:56:29,490 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:29,490 INFO [master/579b0f681375:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-01T23:56:29,493 INFO [master/579b0f681375:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-12-01T23:56:29,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:29,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:29,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:29,508 INFO [master/579b0f681375:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T23:56:29,509 INFO [master/579b0f681375:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-01T23:56:29,510 INFO [master/579b0f681375:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:56:29,533 INFO [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store 2024-12-01T23:56:29,541 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:56:29,542 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T23:56:29,542 INFO [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:56:29,542 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:56:29,542 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T23:56:29,542 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:56:29,542 INFO [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:56:29,542 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733097389541Disabling compacts and flushes for region at 1733097389541Disabling writes for close at 1733097389542 (+1 ms)Writing region close event to WAL at 1733097389542Closed at 1733097389542 2024-12-01T23:56:29,542 WARN [master/579b0f681375:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/.initializing 2024-12-01T23:56:29,543 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/WALs/579b0f681375,35987,1733097389226 2024-12-01T23:56:29,550 INFO [master/579b0f681375:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:56:29,554 INFO [master/579b0f681375:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=579b0f681375%2C35987%2C1733097389226, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/WALs/579b0f681375,35987,1733097389226, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/oldWALs, maxLogs=10 2024-12-01T23:56:29,572 INFO [master/579b0f681375:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/WALs/579b0f681375,35987,1733097389226/579b0f681375%2C35987%2C1733097389226.1733097389554 2024-12-01T23:56:29,573 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T23:56:29,573 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:56:29,573 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:56:29,573 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:29,573 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:29,574 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:29,578 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-01T23:56:29,578 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:29,579 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:29,579 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:29,582 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-01T23:56:29,582 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:29,583 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:56:29,583 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:29,586 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-01T23:56:29,586 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:29,586 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:56:29,586 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:29,589 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-01T23:56:29,589 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:29,589 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:56:29,590 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:29,590 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:29,590 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:29,590 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:29,590 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:29,591 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T23:56:29,591 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:56:29,596 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:56:29,597 INFO [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65505210, jitterRate=-0.02389630675315857}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T23:56:29,597 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733097389573Initializing all the Stores at 1733097389574 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097389574Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733097389574Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733097389574Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733097389574Cleaning up temporary data from old regions at 1733097389590 (+16 ms)Region opened successfully at 1733097389597 (+7 ms) 2024-12-01T23:56:29,598 INFO [master/579b0f681375:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-01T23:56:29,605 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc9a941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=579b0f681375/172.17.0.2:0 2024-12-01T23:56:29,607 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-01T23:56:29,607 INFO [master/579b0f681375:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-01T23:56:29,607 INFO [master/579b0f681375:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-01T23:56:29,607 INFO [master/579b0f681375:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-01T23:56:29,607 INFO [master/579b0f681375:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-01T23:56:29,608 INFO [master/579b0f681375:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-01T23:56:29,608 INFO [master/579b0f681375:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-01T23:56:29,610 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-01T23:56:29,611 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-01T23:56:29,621 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-01T23:56:29,621 INFO [master/579b0f681375:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-01T23:56:29,622 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-01T23:56:29,629 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-01T23:56:29,629 INFO [master/579b0f681375:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-01T23:56:29,631 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-01T23:56:29,637 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-01T23:56:29,638 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-01T23:56:29,646 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-01T23:56:29,649 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-01T23:56:29,654 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-01T23:56:29,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T23:56:29,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T23:56:29,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T23:56:29,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:29,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:29,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:29,663 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=579b0f681375,35987,1733097389226, sessionid=0x101949d0f5d0000, setting cluster-up flag (Was=false) 2024-12-01T23:56:29,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:29,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:29,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:29,704 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-01T23:56:29,706 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=579b0f681375,35987,1733097389226 2024-12-01T23:56:29,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:29,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:29,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:29,746 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-01T23:56:29,747 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=579b0f681375,35987,1733097389226 2024-12-01T23:56:29,748 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-01T23:56:29,749 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-01T23:56:29,750 INFO [master/579b0f681375:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-01T23:56:29,750 INFO [master/579b0f681375:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-01T23:56:29,750 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 579b0f681375,35987,1733097389226 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-01T23:56:29,752 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/579b0f681375:0, corePoolSize=5, maxPoolSize=5 2024-12-01T23:56:29,752 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/579b0f681375:0, corePoolSize=5, maxPoolSize=5 2024-12-01T23:56:29,752 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/579b0f681375:0, corePoolSize=5, maxPoolSize=5 2024-12-01T23:56:29,753 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/579b0f681375:0, corePoolSize=5, maxPoolSize=5 2024-12-01T23:56:29,753 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/579b0f681375:0, corePoolSize=10, maxPoolSize=10 2024-12-01T23:56:29,753 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,753 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/579b0f681375:0, corePoolSize=2, maxPoolSize=2 2024-12-01T23:56:29,753 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,754 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T23:56:29,755 INFO [master/579b0f681375:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733097419755 2024-12-01T23:56:29,755 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-01T23:56:29,755 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-01T23:56:29,755 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-01T23:56:29,755 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-01T23:56:29,755 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-01T23:56:29,755 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-01T23:56:29,755 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-01T23:56:29,755 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:29,755 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,755 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-01T23:56:29,756 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-01T23:56:29,756 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-01T23:56:29,756 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-01T23:56:29,756 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-01T23:56:29,756 INFO [master/579b0f681375:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-01T23:56:29,756 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/579b0f681375:0:becomeActiveMaster-HFileCleaner.large.0-1733097389756,5,FailOnTimeoutGroup] 2024-12-01T23:56:29,757 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/579b0f681375:0:becomeActiveMaster-HFileCleaner.small.0-1733097389756,5,FailOnTimeoutGroup] 2024-12-01T23:56:29,757 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,757 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-01T23:56:29,757 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,757 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,766 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-01T23:56:29,766 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357 2024-12-01T23:56:29,772 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:56:29,772 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-01T23:56:29,776 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T23:56:29,776 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:29,777 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:29,777 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-01T23:56:29,780 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T23:56:29,780 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:29,780 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:29,780 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-01T23:56:29,783 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T23:56:29,784 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:29,784 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:29,784 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-01T23:56:29,787 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T23:56:29,787 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:29,787 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:29,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T23:56:29,789 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740 2024-12-01T23:56:29,789 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740 2024-12-01T23:56:29,789 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T23:56:29,789 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T23:56:29,789 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T23:56:29,791 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T23:56:29,797 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:56:29,797 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72617989, jitterRate=0.08209235966205597}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T23:56:29,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733097389772Initializing all the Stores at 1733097389772Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097389772Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097389772Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733097389772Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097389772Cleaning up temporary data from old regions at 1733097389789 (+17 ms)Region opened successfully at 1733097389798 (+9 ms) 2024-12-01T23:56:29,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T23:56:29,798 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T23:56:29,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T23:56:29,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T23:56:29,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T23:56:29,799 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T23:56:29,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733097389798Disabling compacts and flushes for region at 1733097389798Disabling writes for close at 1733097389798Writing region close event to WAL at 1733097389799 (+1 ms)Closed at 1733097389799 2024-12-01T23:56:29,800 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T23:56:29,800 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-01T23:56:29,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-01T23:56:29,801 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T23:56:29,802 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-01T23:56:29,825 INFO [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer(746): ClusterId : cdd43f0f-e0bf-4fa4-8762-990849e15fef 2024-12-01T23:56:29,825 INFO [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer(746): ClusterId : cdd43f0f-e0bf-4fa4-8762-990849e15fef 2024-12-01T23:56:29,825 DEBUG [RS:1;579b0f681375:45601 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T23:56:29,825 DEBUG [RS:0;579b0f681375:45041 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T23:56:29,863 DEBUG [RS:0;579b0f681375:45041 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T23:56:29,864 DEBUG [RS:0;579b0f681375:45041 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T23:56:29,864 DEBUG [RS:1;579b0f681375:45601 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T23:56:29,864 DEBUG [RS:1;579b0f681375:45601 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T23:56:29,880 DEBUG [RS:0;579b0f681375:45041 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T23:56:29,881 DEBUG [RS:1;579b0f681375:45601 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T23:56:29,882 DEBUG [RS:0;579b0f681375:45041 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68e36a2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=579b0f681375/172.17.0.2:0 2024-12-01T23:56:29,882 DEBUG [RS:1;579b0f681375:45601 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c76ec67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=579b0f681375/172.17.0.2:0 2024-12-01T23:56:29,896 DEBUG [RS:0;579b0f681375:45041 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;579b0f681375:45041 2024-12-01T23:56:29,896 INFO [RS:0;579b0f681375:45041 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T23:56:29,896 INFO [RS:0;579b0f681375:45041 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T23:56:29,896 DEBUG [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T23:56:29,897 INFO [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer(2659): reportForDuty to master=579b0f681375,35987,1733097389226 with port=45041, startcode=1733097389363 2024-12-01T23:56:29,897 DEBUG [RS:0;579b0f681375:45041 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T23:56:29,899 DEBUG [RS:1;579b0f681375:45601 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;579b0f681375:45601 2024-12-01T23:56:29,899 INFO [RS:1;579b0f681375:45601 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T23:56:29,899 INFO [RS:1;579b0f681375:45601 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T23:56:29,899 DEBUG [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T23:56:29,900 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40391, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T23:56:29,900 INFO [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer(2659): reportForDuty to master=579b0f681375,35987,1733097389226 with port=45601, startcode=1733097389400 2024-12-01T23:56:29,900 DEBUG [RS:1;579b0f681375:45601 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T23:56:29,900 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35987 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 579b0f681375,45041,1733097389363 2024-12-01T23:56:29,901 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35987 {}] master.ServerManager(517): Registering regionserver=579b0f681375,45041,1733097389363 2024-12-01T23:56:29,902 DEBUG [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357 2024-12-01T23:56:29,902 DEBUG [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-01T23:56:29,902 DEBUG [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T23:56:29,903 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35299, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T23:56:29,903 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35987 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 579b0f681375,45601,1733097389400 2024-12-01T23:56:29,903 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35987 {}] master.ServerManager(517): Registering regionserver=579b0f681375,45601,1733097389400 2024-12-01T23:56:29,905 DEBUG [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357 2024-12-01T23:56:29,905 DEBUG [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-01T23:56:29,905 DEBUG [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T23:56:29,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T23:56:29,941 DEBUG [RS:0;579b0f681375:45041 {}] zookeeper.ZKUtil(111): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/579b0f681375,45041,1733097389363 2024-12-01T23:56:29,941 WARN [RS:0;579b0f681375:45041 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T23:56:29,941 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [579b0f681375,45041,1733097389363] 2024-12-01T23:56:29,941 INFO [RS:0;579b0f681375:45041 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:56:29,941 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [579b0f681375,45601,1733097389400] 2024-12-01T23:56:29,941 DEBUG [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45041,1733097389363 2024-12-01T23:56:29,941 DEBUG [RS:1;579b0f681375:45601 {}] zookeeper.ZKUtil(111): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/579b0f681375,45601,1733097389400 2024-12-01T23:56:29,941 WARN [RS:1;579b0f681375:45601 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T23:56:29,941 INFO [RS:1;579b0f681375:45601 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:56:29,941 DEBUG [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45601,1733097389400 2024-12-01T23:56:29,945 INFO [RS:1;579b0f681375:45601 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T23:56:29,945 INFO [RS:0;579b0f681375:45041 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T23:56:29,947 INFO [RS:0;579b0f681375:45041 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T23:56:29,947 INFO [RS:0;579b0f681375:45041 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T23:56:29,947 INFO [RS:0;579b0f681375:45041 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,948 INFO [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T23:56:29,948 INFO [RS:1;579b0f681375:45601 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T23:56:29,949 INFO [RS:0;579b0f681375:45041 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T23:56:29,949 INFO [RS:1;579b0f681375:45601 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T23:56:29,949 INFO [RS:0;579b0f681375:45041 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,949 INFO [RS:1;579b0f681375:45601 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,949 DEBUG [RS:0;579b0f681375:45041 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,949 INFO [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T23:56:29,949 DEBUG [RS:0;579b0f681375:45041 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,949 DEBUG [RS:0;579b0f681375:45041 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,949 DEBUG [RS:0;579b0f681375:45041 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,950 DEBUG [RS:0;579b0f681375:45041 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,950 INFO [RS:1;579b0f681375:45601 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T23:56:29,950 DEBUG [RS:0;579b0f681375:45041 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/579b0f681375:0, corePoolSize=2, maxPoolSize=2 2024-12-01T23:56:29,950 INFO [RS:1;579b0f681375:45601 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,950 DEBUG [RS:0;579b0f681375:45041 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,950 DEBUG [RS:0;579b0f681375:45041 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,950 DEBUG [RS:1;579b0f681375:45601 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,950 DEBUG [RS:0;579b0f681375:45041 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,950 DEBUG [RS:1;579b0f681375:45601 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,950 DEBUG [RS:0;579b0f681375:45041 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,950 DEBUG [RS:1;579b0f681375:45601 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,950 DEBUG [RS:0;579b0f681375:45041 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,950 DEBUG [RS:1;579b0f681375:45601 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,950 DEBUG [RS:1;579b0f681375:45601 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,950 DEBUG [RS:0;579b0f681375:45041 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,950 DEBUG [RS:0;579b0f681375:45041 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/579b0f681375:0, corePoolSize=3, maxPoolSize=3 2024-12-01T23:56:29,950 DEBUG [RS:1;579b0f681375:45601 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/579b0f681375:0, corePoolSize=2, maxPoolSize=2 2024-12-01T23:56:29,951 DEBUG [RS:0;579b0f681375:45041 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/579b0f681375:0, corePoolSize=3, maxPoolSize=3 2024-12-01T23:56:29,951 DEBUG [RS:1;579b0f681375:45601 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,951 DEBUG [RS:1;579b0f681375:45601 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,951 DEBUG [RS:1;579b0f681375:45601 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,951 DEBUG [RS:1;579b0f681375:45601 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,951 DEBUG [RS:1;579b0f681375:45601 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,951 DEBUG [RS:1;579b0f681375:45601 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:29,951 DEBUG [RS:1;579b0f681375:45601 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/579b0f681375:0, corePoolSize=3, maxPoolSize=3 2024-12-01T23:56:29,951 DEBUG [RS:1;579b0f681375:45601 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/579b0f681375:0, corePoolSize=3, maxPoolSize=3 2024-12-01T23:56:29,953 WARN [579b0f681375:35987 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T23:56:29,956 INFO [RS:0;579b0f681375:45041 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,956 INFO [RS:1;579b0f681375:45601 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,956 INFO [RS:1;579b0f681375:45601 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,956 INFO [RS:0;579b0f681375:45041 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,956 INFO [RS:1;579b0f681375:45601 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,956 INFO [RS:0;579b0f681375:45041 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,956 INFO [RS:1;579b0f681375:45601 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,956 INFO [RS:0;579b0f681375:45041 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,956 INFO [RS:1;579b0f681375:45601 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,956 INFO [RS:1;579b0f681375:45601 {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,45601,1733097389400-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T23:56:29,956 INFO [RS:0;579b0f681375:45041 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,956 INFO [RS:0;579b0f681375:45041 {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,45041,1733097389363-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T23:56:29,969 INFO [RS:1;579b0f681375:45601 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T23:56:29,969 INFO [RS:1;579b0f681375:45601 {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,45601,1733097389400-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,969 INFO [RS:1;579b0f681375:45601 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,969 INFO [RS:1;579b0f681375:45601 {}] regionserver.Replication(171): 579b0f681375,45601,1733097389400 started 2024-12-01T23:56:29,972 INFO [RS:0;579b0f681375:45041 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T23:56:29,972 INFO [RS:0;579b0f681375:45041 {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,45041,1733097389363-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,972 INFO [RS:0;579b0f681375:45041 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,972 INFO [RS:0;579b0f681375:45041 {}] regionserver.Replication(171): 579b0f681375,45041,1733097389363 started 2024-12-01T23:56:29,982 INFO [RS:1;579b0f681375:45601 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,982 INFO [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer(1482): Serving as 579b0f681375,45601,1733097389400, RpcServer on 579b0f681375/172.17.0.2:45601, sessionid=0x101949d0f5d0002 2024-12-01T23:56:29,983 DEBUG [RS:1;579b0f681375:45601 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T23:56:29,983 DEBUG [RS:1;579b0f681375:45601 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 579b0f681375,45601,1733097389400 2024-12-01T23:56:29,983 DEBUG [RS:1;579b0f681375:45601 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '579b0f681375,45601,1733097389400' 2024-12-01T23:56:29,983 DEBUG [RS:1;579b0f681375:45601 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T23:56:29,984 DEBUG [RS:1;579b0f681375:45601 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T23:56:29,984 DEBUG [RS:1;579b0f681375:45601 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T23:56:29,985 DEBUG [RS:1;579b0f681375:45601 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T23:56:29,985 DEBUG [RS:1;579b0f681375:45601 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 579b0f681375,45601,1733097389400 2024-12-01T23:56:29,985 DEBUG [RS:1;579b0f681375:45601 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '579b0f681375,45601,1733097389400' 2024-12-01T23:56:29,985 DEBUG [RS:1;579b0f681375:45601 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T23:56:29,985 DEBUG [RS:1;579b0f681375:45601 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T23:56:29,985 INFO [RS:0;579b0f681375:45041 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:29,985 INFO [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer(1482): Serving as 579b0f681375,45041,1733097389363, RpcServer on 579b0f681375/172.17.0.2:45041, sessionid=0x101949d0f5d0001 2024-12-01T23:56:29,986 DEBUG [RS:0;579b0f681375:45041 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T23:56:29,986 DEBUG [RS:0;579b0f681375:45041 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 579b0f681375,45041,1733097389363 2024-12-01T23:56:29,986 DEBUG [RS:0;579b0f681375:45041 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '579b0f681375,45041,1733097389363' 2024-12-01T23:56:29,986 DEBUG [RS:0;579b0f681375:45041 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T23:56:29,986 DEBUG [RS:1;579b0f681375:45601 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T23:56:29,986 INFO [RS:1;579b0f681375:45601 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T23:56:29,986 INFO [RS:1;579b0f681375:45601 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T23:56:29,986 DEBUG [RS:0;579b0f681375:45041 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T23:56:29,986 DEBUG [RS:0;579b0f681375:45041 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T23:56:29,986 DEBUG [RS:0;579b0f681375:45041 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T23:56:29,987 DEBUG [RS:0;579b0f681375:45041 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 579b0f681375,45041,1733097389363 2024-12-01T23:56:29,987 DEBUG [RS:0;579b0f681375:45041 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '579b0f681375,45041,1733097389363' 2024-12-01T23:56:29,987 DEBUG [RS:0;579b0f681375:45041 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T23:56:29,987 DEBUG [RS:0;579b0f681375:45041 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T23:56:29,987 DEBUG [RS:0;579b0f681375:45041 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T23:56:29,987 INFO [RS:0;579b0f681375:45041 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T23:56:29,987 INFO [RS:0;579b0f681375:45041 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T23:56:30,087 INFO [RS:1;579b0f681375:45601 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:56:30,089 INFO [RS:0;579b0f681375:45041 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:56:30,089 INFO [RS:1;579b0f681375:45601 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=579b0f681375%2C45601%2C1733097389400, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45601,1733097389400, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/oldWALs, maxLogs=55 2024-12-01T23:56:30,090 INFO [RS:0;579b0f681375:45041 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=579b0f681375%2C45041%2C1733097389363, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45041,1733097389363, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/oldWALs, maxLogs=55 2024-12-01T23:56:30,111 INFO [RS:1;579b0f681375:45601 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45601,1733097389400/579b0f681375%2C45601%2C1733097389400.1733097390094 2024-12-01T23:56:30,111 INFO [RS:0;579b0f681375:45041 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45041,1733097389363/579b0f681375%2C45041%2C1733097389363.1733097390094 2024-12-01T23:56:30,111 DEBUG [RS:1;579b0f681375:45601 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T23:56:30,113 DEBUG [RS:0;579b0f681375:45041 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T23:56:30,203 DEBUG [579b0f681375:35987 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-01T23:56:30,203 DEBUG [579b0f681375:35987 {}] balancer.BalancerClusterState(204): Hosts are {579b0f681375=0} racks are {/default-rack=0} 2024-12-01T23:56:30,204 DEBUG [579b0f681375:35987 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T23:56:30,204 DEBUG [579b0f681375:35987 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T23:56:30,204 DEBUG [579b0f681375:35987 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T23:56:30,205 DEBUG [579b0f681375:35987 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T23:56:30,205 INFO [579b0f681375:35987 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T23:56:30,205 INFO [579b0f681375:35987 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T23:56:30,205 DEBUG [579b0f681375:35987 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T23:56:30,205 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=579b0f681375,45041,1733097389363 2024-12-01T23:56:30,207 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 579b0f681375,45041,1733097389363, state=OPENING 2024-12-01T23:56:30,229 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-01T23:56:30,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:30,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:30,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:30,238 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T23:56:30,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=579b0f681375,45041,1733097389363}] 2024-12-01T23:56:30,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:30,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:30,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:30,393 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T23:56:30,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43349, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T23:56:30,397 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-01T23:56:30,397 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:56:30,397 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-01T23:56:30,398 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=579b0f681375%2C45041%2C1733097389363.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45041,1733097389363, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/oldWALs, maxLogs=55 2024-12-01T23:56:30,416 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45041,1733097389363/579b0f681375%2C45041%2C1733097389363.meta.1733097390399.meta 2024-12-01T23:56:30,416 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T23:56:30,416 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:56:30,417 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-01T23:56:30,417 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-01T23:56:30,417 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-01T23:56:30,417 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-01T23:56:30,417 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:56:30,417 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-01T23:56:30,417 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-01T23:56:30,418 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-01T23:56:30,419 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T23:56:30,419 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:30,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:30,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-01T23:56:30,420 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T23:56:30,420 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:30,420 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:30,420 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-01T23:56:30,421 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T23:56:30,421 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:30,421 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:30,421 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-01T23:56:30,422 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T23:56:30,422 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:30,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:30,422 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T23:56:30,423 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740 2024-12-01T23:56:30,423 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740 2024-12-01T23:56:30,423 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T23:56:30,423 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T23:56:30,423 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T23:56:30,424 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T23:56:30,424 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71797684, jitterRate=0.06986886262893677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T23:56:30,424 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-01T23:56:30,425 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733097390417Writing region info on filesystem at 1733097390417Initializing all the Stores at 1733097390418 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097390418Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097390418Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733097390418Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097390418Cleaning up temporary data from old regions at 1733097390423 (+5 ms)Running coprocessor post-open hooks at 1733097390424 (+1 ms)Region opened successfully at 1733097390425 (+1 ms) 2024-12-01T23:56:30,425 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733097390392 2024-12-01T23:56:30,428 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-01T23:56:30,428 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-01T23:56:30,428 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=579b0f681375,45041,1733097389363 2024-12-01T23:56:30,429 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 579b0f681375,45041,1733097389363, state=OPEN 2024-12-01T23:56:30,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:56:30,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:56:30,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:56:30,462 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=579b0f681375,45041,1733097389363 2024-12-01T23:56:30,462 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:30,462 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:30,462 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:30,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-01T23:56:30,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=579b0f681375,45041,1733097389363 in 223 msec 2024-12-01T23:56:30,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-01T23:56:30,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 665 msec 2024-12-01T23:56:30,470 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T23:56:30,470 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-01T23:56:30,473 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T23:56:30,473 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=579b0f681375,45041,1733097389363, seqNum=-1] 2024-12-01T23:56:30,474 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T23:56:30,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37389, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T23:56:30,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 732 msec 2024-12-01T23:56:30,482 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733097390482, completionTime=-1 2024-12-01T23:56:30,482 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-01T23:56:30,482 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-01T23:56:30,485 INFO [master/579b0f681375:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-01T23:56:30,485 INFO [master/579b0f681375:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733097450485 2024-12-01T23:56:30,485 INFO [master/579b0f681375:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733097510485 2024-12-01T23:56:30,485 INFO [master/579b0f681375:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-01T23:56:30,486 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,35987,1733097389226-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:30,486 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,35987,1733097389226-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:30,486 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,35987,1733097389226-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:30,486 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-579b0f681375:35987, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:30,486 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:30,487 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:30,490 DEBUG [master/579b0f681375:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-01T23:56:30,493 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.039sec 2024-12-01T23:56:30,493 INFO [master/579b0f681375:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-01T23:56:30,493 INFO [master/579b0f681375:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-01T23:56:30,493 INFO [master/579b0f681375:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-01T23:56:30,493 INFO [master/579b0f681375:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-01T23:56:30,494 INFO [master/579b0f681375:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-01T23:56:30,494 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,35987,1733097389226-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T23:56:30,494 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,35987,1733097389226-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-01T23:56:30,495 DEBUG [master/579b0f681375:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-01T23:56:30,495 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-01T23:56:30,495 INFO [master/579b0f681375:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,35987,1733097389226-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:30,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T23:56:30,525 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 579b0f681375,35987,-1 for getting cluster id 2024-12-01T23:56:30,525 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T23:56:30,528 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cdd43f0f-e0bf-4fa4-8762-990849e15fef' 2024-12-01T23:56:30,528 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T23:56:30,529 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cdd43f0f-e0bf-4fa4-8762-990849e15fef" 2024-12-01T23:56:30,529 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@718c8346, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T23:56:30,529 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [579b0f681375,35987,-1] 2024-12-01T23:56:30,530 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T23:56:30,530 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:30,532 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T23:56:30,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34126543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T23:56:30,535 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T23:56:30,536 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=579b0f681375,45041,1733097389363, seqNum=-1] 2024-12-01T23:56:30,537 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T23:56:30,539 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T23:56:30,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=579b0f681375,35987,1733097389226 2024-12-01T23:56:30,542 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-01T23:56:30,543 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 579b0f681375,35987,1733097389226 2024-12-01T23:56:30,544 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@20fa4c17 2024-12-01T23:56:30,544 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-01T23:56:30,545 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-01T23:56:30,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35987 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733097390542', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T23:56:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35987 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733097390542 2024-12-01T23:56:30,548 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733097390542 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T23:56:30,548 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:30,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35987 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733097390542" procId is: 4 2024-12-01T23:56:30,549 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733097390542 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T23:56:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35987 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T23:56:30,558 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733097390542-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1fc76dfa12baf12f91c6fff7488908b3, NAME => 'testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733097390542', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357 2024-12-01T23:56:30,564 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733097390542-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:56:30,564 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733097390542-pool-0 {}] regionserver.HRegion(1722): Closing 1fc76dfa12baf12f91c6fff7488908b3, disabling compactions & flushes 2024-12-01T23:56:30,564 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733097390542-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:30,564 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733097390542-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:30,564 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733097390542-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. after waiting 0 ms 2024-12-01T23:56:30,564 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733097390542-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:30,564 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733097390542-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:30,564 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733097390542-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1fc76dfa12baf12f91c6fff7488908b3: Waiting for close lock at 1733097390564Disabling compacts and flushes for region at 1733097390564Disabling writes for close at 1733097390564Writing region close event to WAL at 1733097390564Closed at 1733097390564 2024-12-01T23:56:30,565 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733097390542 execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T23:56:30,566 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733097390565"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733097390565"}]},"ts":"1733097390565"} 2024-12-01T23:56:30,568 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T23:56:30,569 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733097390542 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T23:56:30,569 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733097390542","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733097390569"}]},"ts":"1733097390569"} 2024-12-01T23:56:30,572 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733097390542, state=ENABLING in hbase:meta 2024-12-01T23:56:30,572 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {579b0f681375=0} racks are {/default-rack=0} 2024-12-01T23:56:30,572 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T23:56:30,572 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T23:56:30,572 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T23:56:30,572 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T23:56:30,573 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T23:56:30,573 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T23:56:30,573 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T23:56:30,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733097390542, region=1fc76dfa12baf12f91c6fff7488908b3, ASSIGN}] 2024-12-01T23:56:30,574 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733097390542, region=1fc76dfa12baf12f91c6fff7488908b3, ASSIGN 2024-12-01T23:56:30,575 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733097390542, region=1fc76dfa12baf12f91c6fff7488908b3, ASSIGN; state=OFFLINE, location=579b0f681375,45041,1733097389363; forceNewPlan=false, retain=false 2024-12-01T23:56:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35987 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T23:56:30,726 INFO [579b0f681375:35987 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-01T23:56:30,727 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1fc76dfa12baf12f91c6fff7488908b3, regionState=OPENING, regionLocation=579b0f681375,45041,1733097389363 2024-12-01T23:56:30,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733097390542, region=1fc76dfa12baf12f91c6fff7488908b3, ASSIGN because future has completed 2024-12-01T23:56:30,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fc76dfa12baf12f91c6fff7488908b3, server=579b0f681375,45041,1733097389363}] 2024-12-01T23:56:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35987 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T23:56:30,890 INFO [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:30,890 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1fc76dfa12baf12f91c6fff7488908b3, NAME => 'testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:56:30,891 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733097390542 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:30,891 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:56:30,891 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:30,891 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:30,892 INFO [StoreOpener-1fc76dfa12baf12f91c6fff7488908b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:30,895 INFO [StoreOpener-1fc76dfa12baf12f91c6fff7488908b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fc76dfa12baf12f91c6fff7488908b3 columnFamilyName family 2024-12-01T23:56:30,895 DEBUG [StoreOpener-1fc76dfa12baf12f91c6fff7488908b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:30,895 INFO [StoreOpener-1fc76dfa12baf12f91c6fff7488908b3-1 {}] regionserver.HStore(327): Store=1fc76dfa12baf12f91c6fff7488908b3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:56:30,895 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:30,896 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:30,896 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:30,896 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:30,896 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:30,896 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:30,903 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:56:30,904 INFO [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1fc76dfa12baf12f91c6fff7488908b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70284428, jitterRate=0.04731959104537964}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T23:56:30,904 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:30,905 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1fc76dfa12baf12f91c6fff7488908b3: Running coprocessor pre-open hook at 1733097390891Writing region info on filesystem at 1733097390891Initializing all the Stores at 1733097390891Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733097390891Cleaning up temporary data from old regions at 1733097390896 (+5 ms)Running coprocessor post-open hooks at 1733097390904 (+8 ms)Region opened successfully at 1733097390905 (+1 ms) 2024-12-01T23:56:30,906 INFO [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3., pid=6, masterSystemTime=1733097390888 2024-12-01T23:56:30,908 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:30,908 INFO [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:30,909 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1fc76dfa12baf12f91c6fff7488908b3, regionState=OPEN, openSeqNum=2, regionLocation=579b0f681375,45041,1733097389363 2024-12-01T23:56:30,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fc76dfa12baf12f91c6fff7488908b3, server=579b0f681375,45041,1733097389363 because future has completed 2024-12-01T23:56:30,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-01T23:56:30,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1fc76dfa12baf12f91c6fff7488908b3, server=579b0f681375,45041,1733097389363 in 179 msec 2024-12-01T23:56:30,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-01T23:56:30,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733097390542, region=1fc76dfa12baf12f91c6fff7488908b3, ASSIGN in 341 msec 2024-12-01T23:56:30,916 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733097390542 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T23:56:30,916 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733097390542","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733097390916"}]},"ts":"1733097390916"} 2024-12-01T23:56:30,919 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733097390542, state=ENABLED in hbase:meta 2024-12-01T23:56:30,919 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733097390542 execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T23:56:30,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733097390542 in 373 msec 2024-12-01T23:56:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35987 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T23:56:31,180 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733097390542 completed 2024-12-01T23:56:31,180 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733097390542 get assigned. Timeout = 60000ms 2024-12-01T23:56:31,180 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T23:56:31,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733097390542 assigned to meta. Checking AM states. 2024-12-01T23:56:31,186 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T23:56:31,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733097390542 assigned. 2024-12-01T23:56:31,192 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733097390542', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3., hostname=579b0f681375,45041,1733097389363, seqNum=2] 2024-12-01T23:56:31,198 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1fc76dfa12baf12f91c6fff7488908b3 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-01T23:56:31,210 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 423e068fcabd46108788fed5d41778f2 2024-12-01T23:56:31,224 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/.tmp/family/423e068fcabd46108788fed5d41778f2 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-01T23:56:31,224 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/.tmp/family/423e068fcabd46108788fed5d41778f2 2024-12-01T23:56:31,227 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/.tmp/family/423e068fcabd46108788fed5d41778f2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/family/423e068fcabd46108788fed5d41778f2 2024-12-01T23:56:31,229 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/family/423e068fcabd46108788fed5d41778f2, delay=958 ms 2024-12-01T23:56:31,231 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/family/423e068fcabd46108788fed5d41778f2, entries=2, sequenceid=6, filesize=4.9 K 2024-12-01T23:56:31,232 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 1fc76dfa12baf12f91c6fff7488908b3 in 34ms, sequenceid=6, compaction requested=false 2024-12-01T23:56:31,232 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733097390542' 2024-12-01T23:56:31,232 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1fc76dfa12baf12f91c6fff7488908b3: 2024-12-01T23:56:31,406 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T23:56:32,189 DEBUG [hfile-prefetch-1733097388580 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 423e068fcabd46108788fed5d41778f2_0. 2024-12-01T23:56:32,189 DEBUG [hfile-prefetch-1733097388580 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 423e068fcabd46108788fed5d41778f2_117. 2024-12-01T23:56:32,190 DEBUG [hfile-prefetch-1733097388580 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/family/423e068fcabd46108788fed5d41778f2, with total blocks 2, and data blocks 1 2024-12-01T23:56:32,190 DEBUG [hfile-prefetch-1733097388580 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-01T23:56:32,190 DEBUG [hfile-prefetch-1733097388580 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/family/423e068fcabd46108788fed5d41778f2, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-01T23:56:32,190 DEBUG [hfile-prefetch-1733097388580 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/family/423e068fcabd46108788fed5d41778f2. Will sleep 100ms and try the verification again. 2024-12-01T23:56:32,233 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;579b0f681375:45041,5,FailOnTimeoutGroup] 2024-12-01T23:56:32,233 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '579b0f681375,45041,1733097389363' ***** 2024-12-01T23:56:32,233 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-12-01T23:56:32,234 INFO [RS:0;579b0f681375:45041 {}] bucket.BucketCache(1738): Disabling cache 2024-12-01T23:56:32,258 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-01T23:56:32,258 INFO [RS:0;579b0f681375:45041 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/bucket.persistence 2024-12-01T23:56:32,258 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-01T23:56:32,258 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-01T23:56:32,258 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T23:56:32,258 INFO [RS:0;579b0f681375:45041 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-01T23:56:32,259 INFO [regionserver/579b0f681375:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T23:56:32,259 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-01T23:56:32,269 DEBUG [RS:0;579b0f681375:45041 {}] bucket.BucketCache(1487): Thread RS:0;579b0f681375:45041 started persisting bucket cache to file 2024-12-01T23:56:32,269 DEBUG [RS:0;579b0f681375:45041 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T23:56:32,269 DEBUG [RS:0;579b0f681375:45041 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T23:56:32,274 DEBUG [RS:0;579b0f681375:45041 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-01T23:56:32,274 DEBUG [RS:0;579b0f681375:45041 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733097392269 2024-12-01T23:56:32,274 DEBUG [RS:0;579b0f681375:45041 {}] bucket.BucketCache(1509): Thread RS:0;579b0f681375:45041 finished persisting bucket cache to file, renaming 2024-12-01T23:56:32,275 INFO [RS:0;579b0f681375:45041 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T23:56:32,275 INFO [RS:0;579b0f681375:45041 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T23:56:32,275 INFO [RS:0;579b0f681375:45041 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T23:56:32,275 INFO [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer(3091): Received CLOSE for 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:32,275 INFO [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer(959): stopping server 579b0f681375,45041,1733097389363 2024-12-01T23:56:32,275 INFO [RS:0;579b0f681375:45041 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T23:56:32,275 INFO [RS:0;579b0f681375:45041 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;579b0f681375:45041. 2024-12-01T23:56:32,275 INFO [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-01T23:56:32,275 DEBUG [RS:0;579b0f681375:45041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T23:56:32,275 DEBUG [RS:0;579b0f681375:45041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:32,275 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1fc76dfa12baf12f91c6fff7488908b3, disabling compactions & flushes 2024-12-01T23:56:32,275 INFO [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:32,275 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:32,275 INFO [RS:0;579b0f681375:45041 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T23:56:32,275 INFO [RS:0;579b0f681375:45041 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T23:56:32,275 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. after waiting 0 ms 2024-12-01T23:56:32,275 INFO [RS:0;579b0f681375:45041 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T23:56:32,275 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:32,276 INFO [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-01T23:56:32,276 INFO [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-01T23:56:32,276 DEBUG [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer(1325): Online Regions={1fc76dfa12baf12f91c6fff7488908b3=testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3., 1588230740=hbase:meta,,1.1588230740} 2024-12-01T23:56:32,276 DEBUG [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:32,276 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-01T23:56:32,276 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T23:56:32,276 DEBUG [StoreFileCloser-1fc76dfa12baf12f91c6fff7488908b3-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/family/423e068fcabd46108788fed5d41778f2 2024-12-01T23:56:32,276 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T23:56:32,276 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T23:56:32,276 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T23:56:32,276 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T23:56:32,276 WARN [hfile-prefetch-1733097388580 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/family/423e068fcabd46108788fed5d41778f2, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-01T23:56:32,277 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-12-01T23:56:32,277 DEBUG [hfile-prefetch-1733097388580 {}] hfile.PrefetchExecutor(121): Prefetch completed for 423e068fcabd46108788fed5d41778f2 2024-12-01T23:56:32,280 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-01T23:56:32,281 INFO [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:32,281 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1fc76dfa12baf12f91c6fff7488908b3: Waiting for close lock at 1733097392275Running coprocessor pre-close hooks at 1733097392275Disabling compacts and flushes for region at 1733097392275Disabling writes for close at 1733097392275Writing region close event to WAL at 1733097392276 (+1 ms)Running coprocessor post-close hooks at 1733097392281 (+5 ms)Closed at 1733097392281 2024-12-01T23:56:32,281 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:32,285 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9cbb8a1d14884df9a3b0347069ec5752 2024-12-01T23:56:32,298 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/info/9cbb8a1d14884df9a3b0347069ec5752 is 209, key is testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3./info:regioninfo/1733097390908/Put/seqid=0 2024-12-01T23:56:32,299 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/info/9cbb8a1d14884df9a3b0347069ec5752 2024-12-01T23:56:32,305 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 25a2004435c044118824b782a0a1b249 2024-12-01T23:56:32,317 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/ns/25a2004435c044118824b782a0a1b249 is 43, key is default/ns:d/1733097390476/Put/seqid=0 2024-12-01T23:56:32,317 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-01T23:56:32,317 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/ns/25a2004435c044118824b782a0a1b249 2024-12-01T23:56:32,317 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-01T23:56:32,317 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T23:56:32,317 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T23:56:32,320 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-01T23:56:32,320 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733097392317 2024-12-01T23:56:32,320 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-01T23:56:32,324 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 52f3105d09e6426e897f430fa2fa842a 2024-12-01T23:56:32,336 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/table/52f3105d09e6426e897f430fa2fa842a is 80, key is testBlockEvictionOnGracefulStop1733097390542/table:state/1733097390916/Put/seqid=0 2024-12-01T23:56:32,336 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/table/52f3105d09e6426e897f430fa2fa842a 2024-12-01T23:56:32,339 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/info/9cbb8a1d14884df9a3b0347069ec5752 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/info/9cbb8a1d14884df9a3b0347069ec5752 2024-12-01T23:56:32,340 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/info/9cbb8a1d14884df9a3b0347069ec5752, delay=905 ms 2024-12-01T23:56:32,341 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/info/9cbb8a1d14884df9a3b0347069ec5752, entries=10, sequenceid=11, filesize=7.0 K 2024-12-01T23:56:32,341 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/ns/25a2004435c044118824b782a0a1b249 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/ns/25a2004435c044118824b782a0a1b249 2024-12-01T23:56:32,342 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/ns/25a2004435c044118824b782a0a1b249, delay=917 ms 2024-12-01T23:56:32,343 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/ns/25a2004435c044118824b782a0a1b249, entries=2, sequenceid=11, filesize=5.0 K 2024-12-01T23:56:32,343 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/table/52f3105d09e6426e897f430fa2fa842a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/table/52f3105d09e6426e897f430fa2fa842a 2024-12-01T23:56:32,344 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/table/52f3105d09e6426e897f430fa2fa842a, delay=967 ms 2024-12-01T23:56:32,345 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/table/52f3105d09e6426e897f430fa2fa842a, entries=2, sequenceid=11, filesize=5.3 K 2024-12-01T23:56:32,346 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 70ms, sequenceid=11, compaction requested=false 2024-12-01T23:56:32,347 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/info/9cbb8a1d14884df9a3b0347069ec5752 2024-12-01T23:56:32,347 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/ns/25a2004435c044118824b782a0a1b249 2024-12-01T23:56:32,348 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/table/52f3105d09e6426e897f430fa2fa842a 2024-12-01T23:56:32,351 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-01T23:56:32,352 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T23:56:32,352 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T23:56:32,352 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733097392276Running coprocessor pre-close hooks at 1733097392276Disabling compacts and flushes for region at 1733097392276Disabling writes for close at 1733097392276Obtaining lock to block concurrent updates at 1733097392277 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733097392277Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1733097392277Flushing stores of hbase:meta,,1.1588230740 at 1733097392277Flushing 1588230740/info: creating writer at 1733097392277Flushing 1588230740/info: appending metadata at 1733097392298 (+21 ms)Flushing 1588230740/info: closing flushed file at 1733097392298Flushing 1588230740/ns: creating writer at 1733097392301 (+3 ms)Flushing 1588230740/ns: appending metadata at 1733097392316 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733097392316Flushing 1588230740/table: creating writer at 1733097392319 (+3 ms)Flushing 1588230740/table: appending metadata at 1733097392335 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733097392335Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39b5a196: reopening flushed file at 1733097392338 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@445b2632: reopening flushed file at 1733097392341 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d647b49: reopening flushed file at 1733097392343 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 70ms, sequenceid=11, compaction requested=false at 1733097392346 (+3 ms)Writing region close event to WAL at 1733097392348 (+2 ms)Running coprocessor post-close hooks at 1733097392352 (+4 ms)Closed at 1733097392352 2024-12-01T23:56:32,352 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-01T23:56:32,476 INFO [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer(976): stopping server 579b0f681375,45041,1733097389363; all regions closed. 2024-12-01T23:56:32,482 DEBUG [RS:0;579b0f681375:45041 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/oldWALs 2024-12-01T23:56:32,482 INFO [RS:0;579b0f681375:45041 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 579b0f681375%2C45041%2C1733097389363.meta:.meta(num 1733097390399) 2024-12-01T23:56:32,485 DEBUG [RS:0;579b0f681375:45041 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/oldWALs 2024-12-01T23:56:32,485 INFO [RS:0;579b0f681375:45041 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 579b0f681375%2C45041%2C1733097389363:(num 1733097390094) 2024-12-01T23:56:32,485 DEBUG [RS:0;579b0f681375:45041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:32,485 INFO [RS:0;579b0f681375:45041 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T23:56:32,485 INFO [RS:0;579b0f681375:45041 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T23:56:32,485 INFO [RS:0;579b0f681375:45041 {}] hbase.ChoreService(370): Chore service for: regionserver/579b0f681375:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-01T23:56:32,486 INFO [RS:0;579b0f681375:45041 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T23:56:32,486 INFO [regionserver/579b0f681375:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T23:56:32,486 INFO [RS:0;579b0f681375:45041 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45041 2024-12-01T23:56:32,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T23:56:32,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/579b0f681375,45041,1733097389363 2024-12-01T23:56:32,521 INFO [RS:0;579b0f681375:45041 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T23:56:32,529 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [579b0f681375,45041,1733097389363] 2024-12-01T23:56:32,537 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/579b0f681375,45041,1733097389363 already deleted, retry=false 2024-12-01T23:56:32,537 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 579b0f681375,45041,1733097389363 on 579b0f681375,35987,1733097389226 2024-12-01T23:56:32,542 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 579b0f681375,45041,1733097389363, splitWal=true, meta=true 2024-12-01T23:56:32,544 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 579b0f681375,45041,1733097389363 (carryingMeta=true) 579b0f681375,45041,1733097389363/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@434191c7[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-01T23:56:32,545 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 579b0f681375,45041,1733097389363, splitWal=true, meta=true 2024-12-01T23:56:32,546 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 579b0f681375,45041,1733097389363, splitWal=true, meta=true, isMeta: true 2024-12-01T23:56:32,546 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45041,1733097389363-splitting 2024-12-01T23:56:32,547 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45041,1733097389363-splitting dir is empty, no logs to split. 2024-12-01T23:56:32,547 INFO [PEWorker-2 {}] master.SplitWALManager(105): 579b0f681375,45041,1733097389363 WAL count=0, meta=true 2024-12-01T23:56:32,549 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45041,1733097389363-splitting dir is empty, no logs to split. 2024-12-01T23:56:32,549 INFO [PEWorker-2 {}] master.SplitWALManager(105): 579b0f681375,45041,1733097389363 WAL count=0, meta=true 2024-12-01T23:56:32,549 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 579b0f681375,45041,1733097389363 WAL splitting is done? wals=0, meta=true 2024-12-01T23:56:32,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-01T23:56:32,553 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T23:56:32,554 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-01T23:56:32,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:56:32,629 INFO [RS:0;579b0f681375:45041 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T23:56:32,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45041-0x101949d0f5d0001, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:56:32,629 INFO [RS:0;579b0f681375:45041 {}] regionserver.HRegionServer(1031): Exiting; stopping=579b0f681375,45041,1733097389363; zookeeper connection closed. 2024-12-01T23:56:32,630 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6c2e141a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6c2e141a 2024-12-01T23:56:32,704 DEBUG [579b0f681375:35987 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-01T23:56:32,704 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=579b0f681375,45601,1733097389400 2024-12-01T23:56:32,705 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 579b0f681375,45601,1733097389400, state=OPENING 2024-12-01T23:56:32,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:56:32,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:56:32,721 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T23:56:32,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:32,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:32,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=579b0f681375,45601,1733097389400}] 2024-12-01T23:56:32,755 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/579b0f681375:0 server-side Connection retries=45 2024-12-01T23:56:32,755 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:32,755 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:32,755 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T23:56:32,755 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:56:32,755 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T23:56:32,755 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T23:56:32,755 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T23:56:32,756 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45445 2024-12-01T23:56:32,757 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45445 connecting to ZooKeeper ensemble=127.0.0.1:56584 2024-12-01T23:56:32,757 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:32,757 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:32,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:454450x0, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T23:56:32,770 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45445-0x101949d0f5d0003 connected 2024-12-01T23:56:32,770 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45445-0x101949d0f5d0003, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-01T23:56:32,770 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-01T23:56:32,771 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T23:56:32,771 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/bucket.cache 2024-12-01T23:56:32,771 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-01T23:56:32,771 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-01T23:56:32,771 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-01T23:56:32,771 INFO [Thread-208 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-01T23:56:32,772 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-01T23:56:32,772 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T23:56:32,772 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45445-0x101949d0f5d0003, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-01T23:56:32,772 INFO [Thread-208 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-01T23:56:32,773 INFO [Thread-208 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-01T23:56:32,774 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45445-0x101949d0f5d0003, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T23:56:32,774 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45445 2024-12-01T23:56:32,777 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45445 2024-12-01T23:56:32,777 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45445 2024-12-01T23:56:32,778 DEBUG [Thread-208 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-01T23:56:32,778 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45445 2024-12-01T23:56:32,778 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45445 2024-12-01T23:56:32,778 INFO [Thread-208 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-01T23:56:32,778 INFO [Thread-208 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-01T23:56:32,778 INFO [Thread-208 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/bucket.persistence is complete. 2024-12-01T23:56:32,784 INFO [RS:2;579b0f681375:45445 {}] regionserver.HRegionServer(746): ClusterId : cdd43f0f-e0bf-4fa4-8762-990849e15fef 2024-12-01T23:56:32,784 DEBUG [RS:2;579b0f681375:45445 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T23:56:32,797 DEBUG [RS:2;579b0f681375:45445 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T23:56:32,797 DEBUG [RS:2;579b0f681375:45445 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T23:56:32,805 DEBUG [RS:2;579b0f681375:45445 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T23:56:32,805 DEBUG [RS:2;579b0f681375:45445 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ebfb7a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=579b0f681375/172.17.0.2:0 2024-12-01T23:56:32,815 DEBUG [RS:2;579b0f681375:45445 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;579b0f681375:45445 2024-12-01T23:56:32,816 INFO [RS:2;579b0f681375:45445 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T23:56:32,816 INFO [RS:2;579b0f681375:45445 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T23:56:32,816 DEBUG [RS:2;579b0f681375:45445 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T23:56:32,817 INFO [RS:2;579b0f681375:45445 {}] regionserver.HRegionServer(2659): reportForDuty to master=579b0f681375,35987,1733097389226 with port=45445, startcode=1733097392755 2024-12-01T23:56:32,817 DEBUG [RS:2;579b0f681375:45445 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T23:56:32,819 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45287, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T23:56:32,819 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35987 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 579b0f681375,45445,1733097392755 2024-12-01T23:56:32,819 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35987 {}] master.ServerManager(517): Registering regionserver=579b0f681375,45445,1733097392755 2024-12-01T23:56:32,820 DEBUG [RS:2;579b0f681375:45445 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357 2024-12-01T23:56:32,820 DEBUG [RS:2;579b0f681375:45445 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-01T23:56:32,820 DEBUG [RS:2;579b0f681375:45445 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T23:56:32,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T23:56:32,829 DEBUG [RS:2;579b0f681375:45445 {}] zookeeper.ZKUtil(111): regionserver:45445-0x101949d0f5d0003, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/579b0f681375,45445,1733097392755 2024-12-01T23:56:32,830 WARN [RS:2;579b0f681375:45445 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T23:56:32,830 INFO [RS:2;579b0f681375:45445 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:56:32,830 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [579b0f681375,45445,1733097392755] 2024-12-01T23:56:32,830 DEBUG [RS:2;579b0f681375:45445 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45445,1733097392755 2024-12-01T23:56:32,834 INFO [RS:2;579b0f681375:45445 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T23:56:32,836 INFO [RS:2;579b0f681375:45445 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T23:56:32,836 INFO [RS:2;579b0f681375:45445 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T23:56:32,836 INFO [RS:2;579b0f681375:45445 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:32,836 INFO [RS:2;579b0f681375:45445 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T23:56:32,837 INFO [RS:2;579b0f681375:45445 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T23:56:32,837 INFO [RS:2;579b0f681375:45445 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:32,837 DEBUG [RS:2;579b0f681375:45445 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:32,837 DEBUG [RS:2;579b0f681375:45445 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:32,837 DEBUG [RS:2;579b0f681375:45445 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:32,837 DEBUG [RS:2;579b0f681375:45445 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:32,837 DEBUG [RS:2;579b0f681375:45445 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:32,837 DEBUG [RS:2;579b0f681375:45445 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/579b0f681375:0, corePoolSize=2, maxPoolSize=2 2024-12-01T23:56:32,837 DEBUG [RS:2;579b0f681375:45445 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:32,837 DEBUG [RS:2;579b0f681375:45445 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:32,837 DEBUG [RS:2;579b0f681375:45445 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:32,838 DEBUG [RS:2;579b0f681375:45445 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:32,838 DEBUG [RS:2;579b0f681375:45445 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:32,838 DEBUG [RS:2;579b0f681375:45445 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/579b0f681375:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:56:32,838 DEBUG [RS:2;579b0f681375:45445 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/579b0f681375:0, corePoolSize=3, maxPoolSize=3 2024-12-01T23:56:32,838 DEBUG [RS:2;579b0f681375:45445 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/579b0f681375:0, corePoolSize=3, maxPoolSize=3 2024-12-01T23:56:32,838 INFO [RS:2;579b0f681375:45445 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:32,838 INFO [RS:2;579b0f681375:45445 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:32,838 INFO [RS:2;579b0f681375:45445 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:32,838 INFO [RS:2;579b0f681375:45445 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:32,838 INFO [RS:2;579b0f681375:45445 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:32,838 INFO [RS:2;579b0f681375:45445 {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,45445,1733097392755-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T23:56:32,851 INFO [RS:2;579b0f681375:45445 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T23:56:32,851 INFO [RS:2;579b0f681375:45445 {}] hbase.ChoreService(168): Chore ScheduledChore name=579b0f681375,45445,1733097392755-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:32,852 INFO [RS:2;579b0f681375:45445 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:32,852 INFO [RS:2;579b0f681375:45445 {}] regionserver.Replication(171): 579b0f681375,45445,1733097392755 started 2024-12-01T23:56:32,864 INFO [RS:2;579b0f681375:45445 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:56:32,864 INFO [RS:2;579b0f681375:45445 {}] regionserver.HRegionServer(1482): Serving as 579b0f681375,45445,1733097392755, RpcServer on 579b0f681375/172.17.0.2:45445, sessionid=0x101949d0f5d0003 2024-12-01T23:56:32,864 DEBUG [RS:2;579b0f681375:45445 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T23:56:32,864 DEBUG [RS:2;579b0f681375:45445 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 579b0f681375,45445,1733097392755 2024-12-01T23:56:32,864 DEBUG [RS:2;579b0f681375:45445 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '579b0f681375,45445,1733097392755' 2024-12-01T23:56:32,864 DEBUG [RS:2;579b0f681375:45445 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T23:56:32,865 DEBUG [RS:2;579b0f681375:45445 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T23:56:32,866 DEBUG [RS:2;579b0f681375:45445 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T23:56:32,866 DEBUG [RS:2;579b0f681375:45445 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T23:56:32,866 DEBUG [RS:2;579b0f681375:45445 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 579b0f681375,45445,1733097392755 2024-12-01T23:56:32,866 DEBUG [RS:2;579b0f681375:45445 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '579b0f681375,45445,1733097392755' 2024-12-01T23:56:32,866 DEBUG [RS:2;579b0f681375:45445 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T23:56:32,866 DEBUG [RS:2;579b0f681375:45445 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T23:56:32,867 DEBUG [RS:2;579b0f681375:45445 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T23:56:32,867 INFO [RS:2;579b0f681375:45445 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T23:56:32,867 INFO [RS:2;579b0f681375:45445 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T23:56:32,874 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T23:56:32,876 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56633, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T23:56:32,877 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-01T23:56:32,877 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:56:32,878 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-01T23:56:32,878 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=579b0f681375%2C45601%2C1733097389400.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45601,1733097389400, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/oldWALs, maxLogs=55 2024-12-01T23:56:32,895 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45601,1733097389400/579b0f681375%2C45601%2C1733097389400.meta.1733097392879.meta 2024-12-01T23:56:32,895 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T23:56:32,895 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:56:32,895 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-01T23:56:32,895 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-01T23:56:32,896 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-01T23:56:32,896 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-01T23:56:32,896 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:56:32,896 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-01T23:56:32,896 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-01T23:56:32,896 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-01T23:56:32,897 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T23:56:32,897 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:32,898 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/info/9cbb8a1d14884df9a3b0347069ec5752, delay=967 ms 2024-12-01T23:56:32,899 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/info/9cbb8a1d14884df9a3b0347069ec5752 2024-12-01T23:56:32,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:32,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-01T23:56:32,900 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T23:56:32,900 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:32,901 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/ns/25a2004435c044118824b782a0a1b249, delay=934 ms 2024-12-01T23:56:32,902 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/ns/25a2004435c044118824b782a0a1b249 2024-12-01T23:56:32,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:32,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-01T23:56:32,903 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T23:56:32,903 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:32,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:32,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-01T23:56:32,903 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T23:56:32,903 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:32,904 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/table/52f3105d09e6426e897f430fa2fa842a, delay=902 ms 2024-12-01T23:56:32,905 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/table/52f3105d09e6426e897f430fa2fa842a 2024-12-01T23:56:32,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:56:32,905 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T23:56:32,906 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740 2024-12-01T23:56:32,906 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740 2024-12-01T23:56:32,907 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T23:56:32,907 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T23:56:32,907 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T23:56:32,908 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T23:56:32,909 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74605694, jitterRate=0.11171147227287292}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T23:56:32,909 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-01T23:56:32,910 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733097392896Writing region info on filesystem at 1733097392896Initializing all the Stores at 1733097392896Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097392896Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097392896Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733097392896Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733097392896Cleaning up temporary data from old regions at 1733097392907 (+11 ms)Running coprocessor post-open hooks at 1733097392909 (+2 ms)Region opened successfully at 1733097392910 (+1 ms) 2024-12-01T23:56:32,911 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1733097392874 2024-12-01T23:56:32,913 DEBUG [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-01T23:56:32,913 INFO [RS_OPEN_META-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-01T23:56:32,913 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=579b0f681375,45601,1733097389400 2024-12-01T23:56:32,914 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 579b0f681375,45601,1733097389400, state=OPEN 2024-12-01T23:56:32,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:56:32,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:56:32,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45445-0x101949d0f5d0003, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:56:32,937 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=579b0f681375,45601,1733097389400 2024-12-01T23:56:32,937 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:32,937 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:32,937 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:56:32,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-01T23:56:32,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=579b0f681375,45601,1733097389400 in 216 msec 2024-12-01T23:56:32,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-01T23:56:32,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 389 msec 2024-12-01T23:56:32,943 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 579b0f681375,45041,1733097389363 had 2 regions 2024-12-01T23:56:32,944 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 579b0f681375,45041,1733097389363, splitWal=true, meta=true, isMeta: false 2024-12-01T23:56:32,945 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45041,1733097389363-splitting dir is empty, no logs to split. 2024-12-01T23:56:32,945 INFO [PEWorker-3 {}] master.SplitWALManager(105): 579b0f681375,45041,1733097389363 WAL count=0, meta=false 2024-12-01T23:56:32,946 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45041,1733097389363-splitting dir is empty, no logs to split. 2024-12-01T23:56:32,947 INFO [PEWorker-3 {}] master.SplitWALManager(105): 579b0f681375,45041,1733097389363 WAL count=0, meta=false 2024-12-01T23:56:32,947 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 579b0f681375,45041,1733097389363 WAL splitting is done? wals=0, meta=false 2024-12-01T23:56:32,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733097390542, region=1fc76dfa12baf12f91c6fff7488908b3, ASSIGN}] 2024-12-01T23:56:32,950 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733097390542, region=1fc76dfa12baf12f91c6fff7488908b3, ASSIGN 2024-12-01T23:56:32,951 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733097390542, region=1fc76dfa12baf12f91c6fff7488908b3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-01T23:56:32,968 INFO [RS:2;579b0f681375:45445 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:56:32,969 INFO [RS:2;579b0f681375:45445 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=579b0f681375%2C45445%2C1733097392755, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45445,1733097392755, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/oldWALs, maxLogs=55 2024-12-01T23:56:32,988 INFO [RS:2;579b0f681375:45445 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/WALs/579b0f681375,45445,1733097392755/579b0f681375%2C45445%2C1733097392755.1733097392970 2024-12-01T23:56:32,991 DEBUG [RS:2;579b0f681375:45445 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-01T23:56:33,102 DEBUG [579b0f681375:35987 {}] balancer.BalancerClusterState(204): Hosts are {579b0f681375=0} racks are {/default-rack=0} 2024-12-01T23:56:33,102 DEBUG [579b0f681375:35987 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T23:56:33,102 DEBUG [579b0f681375:35987 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T23:56:33,102 DEBUG [579b0f681375:35987 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T23:56:33,102 DEBUG [579b0f681375:35987 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T23:56:33,102 INFO [579b0f681375:35987 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T23:56:33,103 INFO [579b0f681375:35987 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T23:56:33,103 DEBUG [579b0f681375:35987 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T23:56:33,103 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1fc76dfa12baf12f91c6fff7488908b3, regionState=OPENING, regionLocation=579b0f681375,45601,1733097389400 2024-12-01T23:56:33,105 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 579b0f681375:45041 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 579b0f681375/172.17.0.2:45041 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:56:33,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=579b0f681375,45041,1733097389363, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=579b0f681375,45041,1733097389363, seqNum=-1, error=java.net.ConnectException: Call to address=579b0f681375:45041 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 579b0f681375/172.17.0.2:45041 2024-12-01T23:56:33,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=579b0f681375,45041,1733097389363, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-01T23:56:33,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=579b0f681375,45041,1733097389363, seqNum=-1 from cache 2024-12-01T23:56:33,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address 579b0f681375:45041 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 579b0f681375/172.17.0.2:45041 2024-12-01T23:56:33,218 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T23:56:33,218 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=579b0f681375,45601,1733097389400, seqNum=-1] 2024-12-01T23:56:33,219 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T23:56:33,220 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51421, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T23:56:33,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733097390542, region=1fc76dfa12baf12f91c6fff7488908b3, ASSIGN because future has completed 2024-12-01T23:56:33,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1fc76dfa12baf12f91c6fff7488908b3, server=579b0f681375,45601,1733097389400}] 2024-12-01T23:56:33,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-01T23:56:33,365 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T23:56:33,365 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T23:56:33,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:33,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:33,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T23:56:33,366 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-01T23:56:33,367 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=618753851, stopped=false 2024-12-01T23:56:33,367 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=579b0f681375,35987,1733097389226 2024-12-01T23:56:33,379 INFO [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:33,379 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 1fc76dfa12baf12f91c6fff7488908b3, NAME => 'testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:56:33,381 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733097390542 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:33,381 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:56:33,381 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:33,381 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:33,382 INFO [StoreOpener-1fc76dfa12baf12f91c6fff7488908b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:33,383 INFO [StoreOpener-1fc76dfa12baf12f91c6fff7488908b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fc76dfa12baf12f91c6fff7488908b3 columnFamilyName family 2024-12-01T23:56:33,384 DEBUG [StoreOpener-1fc76dfa12baf12f91c6fff7488908b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:56:33,386 DEBUG [StoreFileOpener-1fc76dfa12baf12f91c6fff7488908b3-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/family/423e068fcabd46108788fed5d41778f2, delay=979 ms 2024-12-01T23:56:33,388 DEBUG [StoreOpener-1fc76dfa12baf12f91c6fff7488908b3-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/family/423e068fcabd46108788fed5d41778f2 2024-12-01T23:56:33,388 INFO [StoreOpener-1fc76dfa12baf12f91c6fff7488908b3-1 {}] regionserver.HStore(327): Store=1fc76dfa12baf12f91c6fff7488908b3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:56:33,388 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:33,390 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:33,390 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:33,391 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:33,391 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:33,393 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:33,394 INFO [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 1fc76dfa12baf12f91c6fff7488908b3; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63975972, jitterRate=-0.04668372869491577}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T23:56:33,394 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:33,395 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 1fc76dfa12baf12f91c6fff7488908b3: Running coprocessor pre-open hook at 1733097393381Writing region info on filesystem at 1733097393381Initializing all the Stores at 1733097393382 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733097393382Cleaning up temporary data from old regions at 1733097393391 (+9 ms)Running coprocessor post-open hooks at 1733097393394 (+3 ms)Region opened successfully at 1733097393395 (+1 ms) 2024-12-01T23:56:33,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T23:56:33,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T23:56:33,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45445-0x101949d0f5d0003, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T23:56:33,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:33,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45445-0x101949d0f5d0003, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:33,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:33,396 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T23:56:33,396 INFO [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3., pid=11, masterSystemTime=1733097393375 2024-12-01T23:56:33,396 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T23:56:33,397 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T23:56:33,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:33,397 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45445-0x101949d0f5d0003, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:56:33,397 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:56:33,397 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '579b0f681375,45601,1733097389400' ***** 2024-12-01T23:56:33,397 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-01T23:56:33,397 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '579b0f681375,45445,1733097392755' ***** 2024-12-01T23:56:33,397 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-01T23:56:33,398 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:56:33,398 INFO [RS:1;579b0f681375:45601 {}] bucket.BucketCache(1738): Disabling cache 2024-12-01T23:56:33,398 INFO [RS:2;579b0f681375:45445 {}] bucket.BucketCache(1738): Disabling cache 2024-12-01T23:56:33,398 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-01T23:56:33,398 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-01T23:56:33,398 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-01T23:56:33,399 INFO [RS:1;579b0f681375:45601 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/bucket.persistence 2024-12-01T23:56:33,399 INFO [RS:1;579b0f681375:45601 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-01T23:56:33,399 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-01T23:56:33,399 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-01T23:56:33,399 INFO [RS:2;579b0f681375:45445 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fb09dc9f-6b84-9e40-ccd0-3bf2389f2d66/bucket.persistence 2024-12-01T23:56:33,399 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-01T23:56:33,399 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-01T23:56:33,399 INFO [RS:2;579b0f681375:45445 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-01T23:56:33,400 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-01T23:56:33,400 DEBUG [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:33,400 INFO [RS_OPEN_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:33,409 DEBUG [RS:1;579b0f681375:45601 {}] bucket.BucketCache(1487): Thread RS:1;579b0f681375:45601 started persisting bucket cache to file 2024-12-01T23:56:33,409 DEBUG [RS:1;579b0f681375:45601 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T23:56:33,409 DEBUG [RS:1;579b0f681375:45601 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T23:56:33,410 DEBUG [RS:2;579b0f681375:45445 {}] bucket.BucketCache(1487): Thread RS:2;579b0f681375:45445 started persisting bucket cache to file 2024-12-01T23:56:33,410 DEBUG [RS:2;579b0f681375:45445 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-01T23:56:33,410 DEBUG [RS:2;579b0f681375:45445 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-01T23:56:33,412 DEBUG [RS:1;579b0f681375:45601 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-01T23:56:33,412 DEBUG [RS:1;579b0f681375:45601 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733097393409 2024-12-01T23:56:33,412 DEBUG [RS:1;579b0f681375:45601 {}] bucket.BucketCache(1509): Thread RS:1;579b0f681375:45601 finished persisting bucket cache to file, renaming 2024-12-01T23:56:33,412 INFO [RS:1;579b0f681375:45601 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T23:56:33,412 ERROR [RS:2;579b0f681375:45445 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1056) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:56:33,412 INFO [RS:1;579b0f681375:45601 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T23:56:33,412 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T23:56:33,412 INFO [RS:1;579b0f681375:45601 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T23:56:33,412 DEBUG [RS:2;579b0f681375:45445 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-01T23:56:33,412 INFO [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer(3091): Received CLOSE for 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:33,412 DEBUG [RS:2;579b0f681375:45445 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733097393410 2024-12-01T23:56:33,412 DEBUG [RS:2;579b0f681375:45445 {}] bucket.BucketCache(1509): Thread RS:2;579b0f681375:45445 finished persisting bucket cache to file, renaming 2024-12-01T23:56:33,413 INFO [RS:2;579b0f681375:45445 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T23:56:33,413 INFO [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer(959): stopping server 579b0f681375,45601,1733097389400 2024-12-01T23:56:33,413 INFO [RS:2;579b0f681375:45445 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T23:56:33,413 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T23:56:33,413 INFO [RS:1;579b0f681375:45601 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T23:56:33,413 INFO [RS:2;579b0f681375:45445 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T23:56:33,413 INFO [RS:2;579b0f681375:45445 {}] regionserver.HRegionServer(959): stopping server 579b0f681375,45445,1733097392755 2024-12-01T23:56:33,413 INFO [RS:2;579b0f681375:45445 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T23:56:33,413 INFO [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-01T23:56:33,413 INFO [RS:1;579b0f681375:45601 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;579b0f681375:45601. 2024-12-01T23:56:33,413 INFO [RS:2;579b0f681375:45445 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;579b0f681375:45445. 2024-12-01T23:56:33,413 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1fc76dfa12baf12f91c6fff7488908b3, disabling compactions & flushes 2024-12-01T23:56:33,413 DEBUG [RS:2;579b0f681375:45445 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T23:56:33,413 INFO [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:33,413 DEBUG [RS:2;579b0f681375:45445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:33,413 DEBUG [RS:1;579b0f681375:45601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T23:56:33,413 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:33,413 DEBUG [RS:1;579b0f681375:45601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:33,413 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. after waiting 0 ms 2024-12-01T23:56:33,413 INFO [RS:2;579b0f681375:45445 {}] regionserver.HRegionServer(976): stopping server 579b0f681375,45445,1733097392755; all regions closed. 2024-12-01T23:56:33,413 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:33,413 INFO [RS:1;579b0f681375:45601 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T23:56:33,413 INFO [RS:1;579b0f681375:45601 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T23:56:33,413 INFO [RS:1;579b0f681375:45601 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T23:56:33,413 INFO [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-01T23:56:33,414 INFO [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-01T23:56:33,414 DEBUG [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer(1325): Online Regions={1fc76dfa12baf12f91c6fff7488908b3=testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3., 1588230740=hbase:meta,,1.1588230740} 2024-12-01T23:56:33,414 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-01T23:56:33,414 DEBUG [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1fc76dfa12baf12f91c6fff7488908b3 2024-12-01T23:56:33,414 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T23:56:33,414 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T23:56:33,414 DEBUG [StoreFileCloser-1fc76dfa12baf12f91c6fff7488908b3-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/family/423e068fcabd46108788fed5d41778f2 2024-12-01T23:56:33,414 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T23:56:33,414 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T23:56:33,414 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T23:56:33,414 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=485 B heapSize=1.70 KB 2024-12-01T23:56:33,415 DEBUG [RS:2;579b0f681375:45445 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/oldWALs 2024-12-01T23:56:33,415 INFO [RS:2;579b0f681375:45445 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 579b0f681375%2C45445%2C1733097392755:(num 1733097392970) 2024-12-01T23:56:33,415 DEBUG [RS:2;579b0f681375:45445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:33,415 INFO [RS:2;579b0f681375:45445 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T23:56:33,415 INFO [RS:2;579b0f681375:45445 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T23:56:33,415 INFO [RS:2;579b0f681375:45445 {}] hbase.ChoreService(370): Chore service for: regionserver/579b0f681375:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-01T23:56:33,416 INFO [RS:2;579b0f681375:45445 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T23:56:33,416 INFO [regionserver/579b0f681375:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T23:56:33,416 INFO [RS:2;579b0f681375:45445 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T23:56:33,416 INFO [RS:2;579b0f681375:45445 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T23:56:33,416 INFO [RS:2;579b0f681375:45445 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T23:56:33,416 INFO [RS:2;579b0f681375:45445 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45445 2024-12-01T23:56:33,420 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/default/testBlockEvictionOnGracefulStop1733097390542/1fc76dfa12baf12f91c6fff7488908b3/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-01T23:56:33,420 INFO [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:33,420 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1fc76dfa12baf12f91c6fff7488908b3: Waiting for close lock at 1733097393413Running coprocessor pre-close hooks at 1733097393413Disabling compacts and flushes for region at 1733097393413Disabling writes for close at 1733097393413Writing region close event to WAL at 1733097393416 (+3 ms)Running coprocessor post-close hooks at 1733097393420 (+4 ms)Closed at 1733097393420 2024-12-01T23:56:33,420 DEBUG [RS_CLOSE_REGION-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3. 2024-12-01T23:56:33,422 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6e131a2205d94e0a8c1e0a61b11141c5 2024-12-01T23:56:33,433 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/info/6e131a2205d94e0a8c1e0a61b11141c5 is 209, key is testBlockEvictionOnGracefulStop1733097390542,,1733097390546.1fc76dfa12baf12f91c6fff7488908b3./info:regioninfo/1733097393103/Put/seqid=0 2024-12-01T23:56:33,434 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=485 B at sequenceid=18 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/info/6e131a2205d94e0a8c1e0a61b11141c5 2024-12-01T23:56:33,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45445-0x101949d0f5d0003, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/579b0f681375,45445,1733097392755 2024-12-01T23:56:33,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T23:56:33,434 INFO [RS:2;579b0f681375:45445 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T23:56:33,437 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/.tmp/info/6e131a2205d94e0a8c1e0a61b11141c5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/info/6e131a2205d94e0a8c1e0a61b11141c5 2024-12-01T23:56:33,438 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/info/6e131a2205d94e0a8c1e0a61b11141c5, delay=984 ms 2024-12-01T23:56:33,439 INFO [regionserver/579b0f681375:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T23:56:33,439 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/info/6e131a2205d94e0a8c1e0a61b11141c5, entries=3, sequenceid=18, filesize=5.9 K 2024-12-01T23:56:33,440 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 26ms, sequenceid=18, compaction requested=false 2024-12-01T23:56:33,441 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/info/9cbb8a1d14884df9a3b0347069ec5752 2024-12-01T23:56:33,441 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/info/6e131a2205d94e0a8c1e0a61b11141c5 2024-12-01T23:56:33,441 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/ns/25a2004435c044118824b782a0a1b249 2024-12-01T23:56:33,442 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/table/52f3105d09e6426e897f430fa2fa842a 2024-12-01T23:56:33,445 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [579b0f681375,45445,1733097392755] 2024-12-01T23:56:33,445 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/data/hbase/meta/1588230740/recovered.edits/21.seqid, newMaxSeqId=21, maxSeqId=14 2024-12-01T23:56:33,446 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T23:56:33,446 INFO [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T23:56:33,446 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733097393414Running coprocessor pre-close hooks at 1733097393414Disabling compacts and flushes for region at 1733097393414Disabling writes for close at 1733097393414Obtaining lock to block concurrent updates at 1733097393414Preparing flush snapshotting stores in 1588230740 at 1733097393414Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=485, getHeapSize=1672, getOffHeapSize=0, getCellsCount=3 at 1733097393415 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733097393415Flushing 1588230740/info: creating writer at 1733097393415Flushing 1588230740/info: appending metadata at 1733097393433 (+18 ms)Flushing 1588230740/info: closing flushed file at 1733097393433Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ca9a354: reopening flushed file at 1733097393436 (+3 ms)Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 26ms, sequenceid=18, compaction requested=false at 1733097393440 (+4 ms)Writing region close event to WAL at 1733097393442 (+2 ms)Running coprocessor post-close hooks at 1733097393446 (+4 ms)Closed at 1733097393446 2024-12-01T23:56:33,446 DEBUG [RS_CLOSE_META-regionserver/579b0f681375:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-01T23:56:33,453 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/579b0f681375,45445,1733097392755 already deleted, retry=false 2024-12-01T23:56:33,454 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 579b0f681375,45445,1733097392755 expired; onlineServers=1 2024-12-01T23:56:33,460 INFO [regionserver/579b0f681375:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T23:56:33,477 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-01T23:56:33,478 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-01T23:56:33,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45445-0x101949d0f5d0003, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:56:33,546 INFO [RS:2;579b0f681375:45445 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T23:56:33,546 INFO [RS:2;579b0f681375:45445 {}] regionserver.HRegionServer(1031): Exiting; stopping=579b0f681375,45445,1733097392755; zookeeper connection closed. 2024-12-01T23:56:33,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45445-0x101949d0f5d0003, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:56:33,546 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1703e5a0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1703e5a0 2024-12-01T23:56:33,614 INFO [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer(976): stopping server 579b0f681375,45601,1733097389400; all regions closed. 2024-12-01T23:56:33,616 DEBUG [RS:1;579b0f681375:45601 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/oldWALs 2024-12-01T23:56:33,616 INFO [RS:1;579b0f681375:45601 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 579b0f681375%2C45601%2C1733097389400.meta:.meta(num 1733097392879) 2024-12-01T23:56:33,618 DEBUG [RS:1;579b0f681375:45601 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/oldWALs 2024-12-01T23:56:33,618 INFO [RS:1;579b0f681375:45601 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 579b0f681375%2C45601%2C1733097389400:(num 1733097390094) 2024-12-01T23:56:33,618 DEBUG [RS:1;579b0f681375:45601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:56:33,618 INFO [RS:1;579b0f681375:45601 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T23:56:33,618 INFO [RS:1;579b0f681375:45601 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T23:56:33,619 INFO [RS:1;579b0f681375:45601 {}] hbase.ChoreService(370): Chore service for: regionserver/579b0f681375:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-01T23:56:33,619 INFO [RS:1;579b0f681375:45601 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T23:56:33,619 INFO [regionserver/579b0f681375:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T23:56:33,619 INFO [RS:1;579b0f681375:45601 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45601 2024-12-01T23:56:33,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/579b0f681375,45601,1733097389400 2024-12-01T23:56:33,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T23:56:33,629 INFO [RS:1;579b0f681375:45601 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T23:56:33,637 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [579b0f681375,45601,1733097389400] 2024-12-01T23:56:33,645 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/579b0f681375,45601,1733097389400 already deleted, retry=false 2024-12-01T23:56:33,645 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 579b0f681375,45601,1733097389400 expired; onlineServers=0 2024-12-01T23:56:33,645 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '579b0f681375,35987,1733097389226' ***** 2024-12-01T23:56:33,645 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-01T23:56:33,646 INFO [M:0;579b0f681375:35987 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T23:56:33,646 INFO [M:0;579b0f681375:35987 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T23:56:33,646 DEBUG [M:0;579b0f681375:35987 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-01T23:56:33,646 DEBUG [M:0;579b0f681375:35987 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-01T23:56:33,646 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-01T23:56:33,646 DEBUG [master/579b0f681375:0:becomeActiveMaster-HFileCleaner.large.0-1733097389756 {}] cleaner.HFileCleaner(306): Exit Thread[master/579b0f681375:0:becomeActiveMaster-HFileCleaner.large.0-1733097389756,5,FailOnTimeoutGroup] 2024-12-01T23:56:33,646 DEBUG [master/579b0f681375:0:becomeActiveMaster-HFileCleaner.small.0-1733097389756 {}] cleaner.HFileCleaner(306): Exit Thread[master/579b0f681375:0:becomeActiveMaster-HFileCleaner.small.0-1733097389756,5,FailOnTimeoutGroup] 2024-12-01T23:56:33,646 INFO [M:0;579b0f681375:35987 {}] hbase.ChoreService(370): Chore service for: master/579b0f681375:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-01T23:56:33,646 INFO [M:0;579b0f681375:35987 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T23:56:33,647 DEBUG [M:0;579b0f681375:35987 {}] master.HMaster(1795): Stopping service threads 2024-12-01T23:56:33,647 INFO [M:0;579b0f681375:35987 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-01T23:56:33,647 INFO [M:0;579b0f681375:35987 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T23:56:33,647 INFO [M:0;579b0f681375:35987 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-01T23:56:33,647 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-01T23:56:33,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-01T23:56:33,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:56:33,654 DEBUG [M:0;579b0f681375:35987 {}] zookeeper.ZKUtil(347): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-01T23:56:33,654 WARN [M:0;579b0f681375:35987 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-01T23:56:33,655 INFO [M:0;579b0f681375:35987 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/.lastflushedseqids 2024-12-01T23:56:33,663 INFO [M:0;579b0f681375:35987 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-01T23:56:33,664 INFO [M:0;579b0f681375:35987 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-01T23:56:33,664 DEBUG [M:0;579b0f681375:35987 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T23:56:33,664 INFO [M:0;579b0f681375:35987 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:56:33,664 DEBUG [M:0;579b0f681375:35987 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:56:33,664 DEBUG [M:0;579b0f681375:35987 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T23:56:33,664 DEBUG [M:0;579b0f681375:35987 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:56:33,664 INFO [M:0;579b0f681375:35987 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=35.30 KB heapSize=44.59 KB 2024-12-01T23:56:33,676 DEBUG [M:0;579b0f681375:35987 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d17eab61a19f4b79bf5a3f01817c4623 2024-12-01T23:56:33,691 DEBUG [M:0;579b0f681375:35987 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d17eab61a19f4b79bf5a3f01817c4623 is 82, key is hbase:meta,,1/info:regioninfo/1733097392913/Put/seqid=0 2024-12-01T23:56:33,692 INFO [M:0;579b0f681375:35987 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=91 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d17eab61a19f4b79bf5a3f01817c4623 2024-12-01T23:56:33,699 DEBUG [M:0;579b0f681375:35987 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f778919fadcb4193ac7cce5fe16605ad 2024-12-01T23:56:33,712 DEBUG [M:0;579b0f681375:35987 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f778919fadcb4193ac7cce5fe16605ad is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733097390920/Put/seqid=0 2024-12-01T23:56:33,713 INFO [M:0;579b0f681375:35987 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=91 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f778919fadcb4193ac7cce5fe16605ad 2024-12-01T23:56:33,720 DEBUG [M:0;579b0f681375:35987 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 618d67f510374742ab684c43d49a0556 2024-12-01T23:56:33,732 DEBUG [M:0;579b0f681375:35987 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/618d67f510374742ab684c43d49a0556 is 69, key is 579b0f681375,45445,1733097392755/rs:state/1733097392819/Put/seqid=0 2024-12-01T23:56:33,732 INFO [M:0;579b0f681375:35987 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=91 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/618d67f510374742ab684c43d49a0556 2024-12-01T23:56:33,734 INFO [M:0;579b0f681375:35987 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 618d67f510374742ab684c43d49a0556 2024-12-01T23:56:33,734 DEBUG [M:0;579b0f681375:35987 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d17eab61a19f4b79bf5a3f01817c4623 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d17eab61a19f4b79bf5a3f01817c4623 2024-12-01T23:56:33,736 INFO [M:0;579b0f681375:35987 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d17eab61a19f4b79bf5a3f01817c4623, entries=14, sequenceid=91, filesize=5.9 K 2024-12-01T23:56:33,736 DEBUG [M:0;579b0f681375:35987 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f778919fadcb4193ac7cce5fe16605ad as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f778919fadcb4193ac7cce5fe16605ad 2024-12-01T23:56:33,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:56:33,737 INFO [RS:1;579b0f681375:45601 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T23:56:33,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45601-0x101949d0f5d0002, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:56:33,737 INFO [RS:1;579b0f681375:45601 {}] regionserver.HRegionServer(1031): Exiting; stopping=579b0f681375,45601,1733097389400; zookeeper connection closed. 2024-12-01T23:56:33,737 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7b661e1b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7b661e1b 2024-12-01T23:56:33,738 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-01T23:56:33,738 INFO [M:0;579b0f681375:35987 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f778919fadcb4193ac7cce5fe16605ad, entries=11, sequenceid=91, filesize=7.8 K 2024-12-01T23:56:33,738 DEBUG [M:0;579b0f681375:35987 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/618d67f510374742ab684c43d49a0556 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/618d67f510374742ab684c43d49a0556 2024-12-01T23:56:33,739 INFO [M:0;579b0f681375:35987 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 618d67f510374742ab684c43d49a0556 2024-12-01T23:56:33,739 INFO [M:0;579b0f681375:35987 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/059337a8-0a47-9fc2-55ff-b312f181089d/6fa5fbf3-d899-a9b6-95dc-3bae3f590357/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/618d67f510374742ab684c43d49a0556, entries=3, sequenceid=91, filesize=5.3 K 2024-12-01T23:56:33,740 INFO [M:0;579b0f681375:35987 {}] regionserver.HRegion(3140): Finished flush of dataSize ~35.30 KB/36148, heapSize ~44.30 KB/45360, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=91, compaction requested=false 2024-12-01T23:56:33,742 INFO [M:0;579b0f681375:35987 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:56:33,742 DEBUG [M:0;579b0f681375:35987 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733097393664Disabling compacts and flushes for region at 1733097393664Disabling writes for close at 1733097393664Obtaining lock to block concurrent updates at 1733097393664Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733097393664Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=36148, getHeapSize=45600, getOffHeapSize=0, getCellsCount=112 at 1733097393665 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733097393665Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733097393665Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733097393691 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733097393691Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733097393694 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733097393712 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733097393712Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733097393715 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733097393731 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733097393731Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@600ffb7a: reopening flushed file at 1733097393734 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25ecb9f0: reopening flushed file at 1733097393736 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@705972b: reopening flushed file at 1733097393738 (+2 ms)Finished flush of dataSize ~35.30 KB/36148, heapSize ~44.30 KB/45360, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=91, compaction requested=false at 1733097393740 (+2 ms)Writing region close event to WAL at 1733097393742 (+2 ms)Closed at 1733097393742 2024-12-01T23:56:33,743 INFO [M:0;579b0f681375:35987 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-01T23:56:33,743 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T23:56:33,743 INFO [M:0;579b0f681375:35987 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35987 2024-12-01T23:56:33,743 INFO [M:0;579b0f681375:35987 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T23:56:33,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:56:33,863 INFO [M:0;579b0f681375:35987 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T23:56:33,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35987-0x101949d0f5d0000, quorum=127.0.0.1:56584, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:56:33,869 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-01T23:56:33,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-01T23:56:33,871 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:56:33,877 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 51) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=353 (was 358), ProcessCount=11 (was 11), AvailableMemoryMB=7446 (was 7531)